diff --git a/arch/riscv/Kconfig.debug b/arch/riscv/Kconfig.debug index eafe17ebf7102c..be202267da6db9 100644 --- a/arch/riscv/Kconfig.debug +++ b/arch/riscv/Kconfig.debug @@ -1 +1,2 @@ source "arch/riscv/kernel/tests/Kconfig.debug" +source "arch/riscv/kvm/Kconfig.debug" diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 371da75a47f960..758f7cdc09d62e 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -177,6 +177,9 @@ BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zs all: $(notdir $(KBUILD_IMAGE)) +archprepare: + $(Q)$(MAKE) $(build)=arch/riscv/tools insn + loader.bin: loader Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader xipImage vmlinuz.efi: Image diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index bd5fc940329534..82feba7b0eb5c3 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -2,6 +2,7 @@ syscall-y += syscall_table_32.h syscall-y += syscall_table_64.h +generated-y += insn_gen.h generic-y += early_ioremap.h generic-y += flat.h generic-y += fprobe.h diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index c3005573e8c999..6ea8dc08a29046 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -8,151 +8,52 @@ #include -#define RV_INSN_FUNCT3_MASK GENMASK(14, 12) -#define RV_INSN_FUNCT3_OPOFF 12 -#define RV_INSN_OPCODE_MASK GENMASK(6, 0) -#define RV_INSN_OPCODE_OPOFF 0 -#define RV_INSN_FUNCT12_OPOFF 20 - -#define RV_ENCODE_FUNCT3(f_) (RVG_FUNCT3_##f_ << RV_INSN_FUNCT3_OPOFF) -#define RV_ENCODE_FUNCT12(f_) (RVG_FUNCT12_##f_ << RV_INSN_FUNCT12_OPOFF) - -/* The bit field of immediate value in I-type instruction */ -#define RV_I_IMM_SIGN_OPOFF 31 -#define RV_I_IMM_11_0_OPOFF 20 -#define RV_I_IMM_SIGN_OFF 12 -#define RV_I_IMM_11_0_OFF 0 -#define RV_I_IMM_11_0_MASK GENMASK(11, 0) - -/* The bit field of immediate value in J-type instruction */ -#define RV_J_IMM_SIGN_OPOFF 31 -#define RV_J_IMM_10_1_OPOFF 21 -#define RV_J_IMM_11_OPOFF 20 -#define RV_J_IMM_19_12_OPOFF 12 -#define RV_J_IMM_SIGN_OFF 20 -#define RV_J_IMM_10_1_OFF 1 -#define RV_J_IMM_11_OFF 11 -#define RV_J_IMM_19_12_OFF 12 -#define RV_J_IMM_10_1_MASK GENMASK(9, 0) -#define RV_J_IMM_11_MASK GENMASK(0, 0) -#define RV_J_IMM_19_12_MASK GENMASK(7, 0) - /* - * U-type IMMs contain the upper 20bits [31:20] of an immediate with - * the rest filled in by zeros, so no shifting required. Similarly, - * bit31 contains the signed state, so no sign extension necessary. + * Generate a function to check if a sequence of bits matches an instruction */ -#define RV_U_IMM_SIGN_OPOFF 31 -#define RV_U_IMM_31_12_OPOFF 0 -#define RV_U_IMM_31_12_MASK GENMASK(31, 12) - -/* The bit field of immediate value in B-type instruction */ -#define RV_B_IMM_SIGN_OPOFF 31 -#define RV_B_IMM_10_5_OPOFF 25 -#define RV_B_IMM_4_1_OPOFF 8 -#define RV_B_IMM_11_OPOFF 7 -#define RV_B_IMM_SIGN_OFF 12 -#define RV_B_IMM_10_5_OFF 5 -#define RV_B_IMM_4_1_OFF 1 -#define RV_B_IMM_11_OFF 11 -#define RV_B_IMM_10_5_MASK GENMASK(5, 0) -#define RV_B_IMM_4_1_MASK GENMASK(3, 0) -#define RV_B_IMM_11_MASK GENMASK(0, 0) - -/* The register offset in RVG instruction */ -#define RVG_RS1_OPOFF 15 -#define RVG_RS2_OPOFF 20 -#define RVG_RD_OPOFF 7 -#define RVG_RS1_MASK GENMASK(4, 0) -#define RVG_RS2_MASK GENMASK(4, 0) -#define RVG_RD_MASK GENMASK(4, 0) - -/* The bit field of immediate value in RVC J instruction */ -#define RVC_J_IMM_SIGN_OPOFF 12 -#define RVC_J_IMM_4_OPOFF 11 -#define RVC_J_IMM_9_8_OPOFF 9 -#define RVC_J_IMM_10_OPOFF 8 -#define RVC_J_IMM_6_OPOFF 7 -#define RVC_J_IMM_7_OPOFF 6 -#define RVC_J_IMM_3_1_OPOFF 3 -#define RVC_J_IMM_5_OPOFF 2 -#define RVC_J_IMM_SIGN_OFF 11 -#define RVC_J_IMM_4_OFF 4 -#define RVC_J_IMM_9_8_OFF 8 -#define RVC_J_IMM_10_OFF 10 -#define RVC_J_IMM_6_OFF 6 -#define RVC_J_IMM_7_OFF 7 -#define RVC_J_IMM_3_1_OFF 1 -#define RVC_J_IMM_5_OFF 5 -#define RVC_J_IMM_4_MASK GENMASK(0, 0) -#define RVC_J_IMM_9_8_MASK GENMASK(1, 0) -#define RVC_J_IMM_10_MASK GENMASK(0, 0) -#define RVC_J_IMM_6_MASK GENMASK(0, 0) -#define RVC_J_IMM_7_MASK GENMASK(0, 0) -#define RVC_J_IMM_3_1_MASK GENMASK(2, 0) -#define RVC_J_IMM_5_MASK GENMASK(0, 0) - -/* The bit field of immediate value in RVC B instruction */ -#define RVC_B_IMM_SIGN_OPOFF 12 -#define RVC_B_IMM_4_3_OPOFF 10 -#define RVC_B_IMM_7_6_OPOFF 5 -#define RVC_B_IMM_2_1_OPOFF 3 -#define RVC_B_IMM_5_OPOFF 2 -#define RVC_B_IMM_SIGN_OFF 8 -#define RVC_B_IMM_4_3_OFF 3 -#define RVC_B_IMM_7_6_OFF 6 -#define RVC_B_IMM_2_1_OFF 1 -#define RVC_B_IMM_5_OFF 5 -#define RVC_B_IMM_4_3_MASK GENMASK(1, 0) -#define RVC_B_IMM_7_6_MASK GENMASK(1, 0) -#define RVC_B_IMM_2_1_MASK GENMASK(1, 0) -#define RVC_B_IMM_5_MASK GENMASK(0, 0) +#define __RISCV_INSN_FUNCS(name) \ +static __always_inline bool riscv_insn_is_##name(u32 _insn) \ +{ \ + BUILD_BUG_ON(~(riscv_insn_##name##_MASK) & (riscv_insn_##name##_MATCH)); \ + return (_insn & (riscv_insn_##name##_MASK)) == (riscv_insn_##name##_MATCH); \ +} -#define RVC_INSN_FUNCT4_MASK GENMASK(15, 12) -#define RVC_INSN_FUNCT4_OPOFF 12 -#define RVC_INSN_FUNCT3_MASK GENMASK(15, 13) -#define RVC_INSN_FUNCT3_OPOFF 13 -#define RVC_INSN_J_RS1_MASK GENMASK(11, 7) -#define RVC_INSN_J_RS2_MASK GENMASK(6, 2) -#define RVC_INSN_OPCODE_MASK GENMASK(1, 0) -#define RVC_ENCODE_FUNCT3(f_) (RVC_FUNCT3_##f_ << RVC_INSN_FUNCT3_OPOFF) -#define RVC_ENCODE_FUNCT4(f_) (RVC_FUNCT4_##f_ << RVC_INSN_FUNCT4_OPOFF) +/* + * Generate a function to check if a sequence of bits matches an instruction + * with constraints. Some instructions require inputs to be specific values. + */ +#define __RISCV_INSN_FUNCS_CONSTRAINED(name, constraints) \ +static __always_inline bool riscv_insn_is_##name(u32 _insn) \ +{ \ + BUILD_BUG_ON(~(riscv_insn_##name##_MASK) & (riscv_insn_##name##_MATCH)); \ + return ((_insn & (riscv_insn_##name##_MASK)) == (riscv_insn_##name##_MATCH)) && \ + (constraints); \ +} -/* The register offset in RVC op=C0 instruction */ -#define RVC_C0_RS1_OPOFF 7 -#define RVC_C0_RS2_OPOFF 2 -#define RVC_C0_RD_OPOFF 2 +#define __RISCV_INSN_FUNCS_UNSUPPORTED(name) \ +static __always_inline bool riscv_insn_is_##name(u32 _insn) \ +{ \ + return 0; \ +} -/* The register offset in RVC op=C1 instruction */ -#define RVC_C1_RS1_OPOFF 7 -#define RVC_C1_RS2_OPOFF 2 -#define RVC_C1_RD_OPOFF 7 +#include -/* The register offset in RVC op=C2 instruction */ -#define RVC_C2_RS1_OPOFF 7 -#define RVC_C2_RS2_OPOFF 2 -#define RVC_C2_RD_OPOFF 7 -#define RVC_C2_RS1_MASK GENMASK(4, 0) +#define RV_INSN_OPCODE_MASK GENMASK(6, 0) /* parts of opcode for RVG*/ -#define RVG_OPCODE_FENCE 0x0f -#define RVG_OPCODE_AUIPC 0x17 #define RVG_OPCODE_BRANCH 0x63 -#define RVG_OPCODE_JALR 0x67 -#define RVG_OPCODE_JAL 0x6f #define RVG_OPCODE_SYSTEM 0x73 #define RVG_SYSTEM_CSR_OFF 20 #define RVG_SYSTEM_CSR_MASK GENMASK(12, 0) +// THESE ARE ALL ACTUALLY USED /* parts of opcode for RVF, RVD and RVQ */ #define RVFDQ_FL_FS_WIDTH_OFF 12 #define RVFDQ_FL_FS_WIDTH_MASK GENMASK(2, 0) -#define RVFDQ_FL_FS_WIDTH_W 2 -#define RVFDQ_FL_FS_WIDTH_D 3 -#define RVFDQ_LS_FS_WIDTH_Q 4 #define RVFDQ_OPCODE_FL 0x07 #define RVFDQ_OPCODE_FS 0x27 +// THESE ARE ALL ACTUALLY USED /* parts of opcode for RVV */ #define RVV_OPCODE_VECTOR 0x57 #define RVV_VL_VS_WIDTH_8 0 @@ -162,107 +63,11 @@ #define RVV_OPCODE_VL RVFDQ_OPCODE_FL #define RVV_OPCODE_VS RVFDQ_OPCODE_FS -/* parts of opcode for RVC*/ -#define RVC_OPCODE_C0 0x0 -#define RVC_OPCODE_C1 0x1 -#define RVC_OPCODE_C2 0x2 - -/* parts of funct3 code for I, M, A extension*/ -#define RVG_FUNCT3_JALR 0x0 -#define RVG_FUNCT3_BEQ 0x0 -#define RVG_FUNCT3_BNE 0x1 -#define RVG_FUNCT3_BLT 0x4 -#define RVG_FUNCT3_BGE 0x5 -#define RVG_FUNCT3_BLTU 0x6 -#define RVG_FUNCT3_BGEU 0x7 - -/* parts of funct3 code for C extension*/ -#define RVC_FUNCT3_C_BEQZ 0x6 -#define RVC_FUNCT3_C_BNEZ 0x7 -#define RVC_FUNCT3_C_J 0x5 -#define RVC_FUNCT3_C_JAL 0x1 -#define RVC_FUNCT4_C_JR 0x8 -#define RVC_FUNCT4_C_JALR 0x9 -#define RVC_FUNCT4_C_EBREAK 0x9 - -#define RVG_FUNCT12_EBREAK 0x1 -#define RVG_FUNCT12_SRET 0x102 - -#define RVG_MATCH_AUIPC (RVG_OPCODE_AUIPC) -#define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR) -#define RVG_MATCH_JAL (RVG_OPCODE_JAL) -#define RVG_MATCH_FENCE (RVG_OPCODE_FENCE) -#define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BNE (RV_ENCODE_FUNCT3(BNE) | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BLT (RV_ENCODE_FUNCT3(BLT) | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BGE (RV_ENCODE_FUNCT3(BGE) | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BLTU (RV_ENCODE_FUNCT3(BLTU) | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BGEU (RV_ENCODE_FUNCT3(BGEU) | RVG_OPCODE_BRANCH) -#define RVG_MATCH_EBREAK (RV_ENCODE_FUNCT12(EBREAK) | RVG_OPCODE_SYSTEM) -#define RVG_MATCH_SRET (RV_ENCODE_FUNCT12(SRET) | RVG_OPCODE_SYSTEM) -#define RVC_MATCH_C_BEQZ (RVC_ENCODE_FUNCT3(C_BEQZ) | RVC_OPCODE_C1) -#define RVC_MATCH_C_BNEZ (RVC_ENCODE_FUNCT3(C_BNEZ) | RVC_OPCODE_C1) -#define RVC_MATCH_C_J (RVC_ENCODE_FUNCT3(C_J) | RVC_OPCODE_C1) -#define RVC_MATCH_C_JAL (RVC_ENCODE_FUNCT3(C_JAL) | RVC_OPCODE_C1) -#define RVC_MATCH_C_JR (RVC_ENCODE_FUNCT4(C_JR) | RVC_OPCODE_C2) -#define RVC_MATCH_C_JALR (RVC_ENCODE_FUNCT4(C_JALR) | RVC_OPCODE_C2) -#define RVC_MATCH_C_EBREAK (RVC_ENCODE_FUNCT4(C_EBREAK) | RVC_OPCODE_C2) - -#define RVG_MASK_AUIPC (RV_INSN_OPCODE_MASK) -#define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) -#define RVG_MASK_FENCE (RV_INSN_OPCODE_MASK) -#define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) -#define RVC_MASK_C_JR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) -#define RVC_MASK_C_JAL (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) -#define RVC_MASK_C_J (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) -#define RVG_MASK_BEQ (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVG_MASK_BNE (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVG_MASK_BLT (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVG_MASK_BGE (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVG_MASK_BLTU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVG_MASK_BGEU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) -#define RVC_MASK_C_BEQZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) -#define RVC_MASK_C_BNEZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) -#define RVC_MASK_C_EBREAK 0xffff -#define RVG_MASK_EBREAK 0xffffffff -#define RVG_MASK_SRET 0xffffffff - #define __INSN_LENGTH_MASK _UL(0x3) #define __INSN_LENGTH_GE_32 _UL(0x3) #define __INSN_OPCODE_MASK _UL(0x7F) #define __INSN_BRANCH_OPCODE _UL(RVG_OPCODE_BRANCH) -#define __RISCV_INSN_FUNCS(name, mask, val) \ -static __always_inline bool riscv_insn_is_##name(u32 code) \ -{ \ - BUILD_BUG_ON(~(mask) & (val)); \ - return (code & (mask)) == (val); \ -} \ - -#if __riscv_xlen == 32 -/* C.JAL is an RV32C-only instruction */ -__RISCV_INSN_FUNCS(c_jal, RVC_MASK_C_JAL, RVC_MATCH_C_JAL) -#else -#define riscv_insn_is_c_jal(opcode) 0 -#endif -__RISCV_INSN_FUNCS(auipc, RVG_MASK_AUIPC, RVG_MATCH_AUIPC) -__RISCV_INSN_FUNCS(jalr, RVG_MASK_JALR, RVG_MATCH_JALR) -__RISCV_INSN_FUNCS(jal, RVG_MASK_JAL, RVG_MATCH_JAL) -__RISCV_INSN_FUNCS(c_j, RVC_MASK_C_J, RVC_MATCH_C_J) -__RISCV_INSN_FUNCS(beq, RVG_MASK_BEQ, RVG_MATCH_BEQ) -__RISCV_INSN_FUNCS(bne, RVG_MASK_BNE, RVG_MATCH_BNE) -__RISCV_INSN_FUNCS(blt, RVG_MASK_BLT, RVG_MATCH_BLT) -__RISCV_INSN_FUNCS(bge, RVG_MASK_BGE, RVG_MATCH_BGE) -__RISCV_INSN_FUNCS(bltu, RVG_MASK_BLTU, RVG_MATCH_BLTU) -__RISCV_INSN_FUNCS(bgeu, RVG_MASK_BGEU, RVG_MATCH_BGEU) -__RISCV_INSN_FUNCS(c_beqz, RVC_MASK_C_BEQZ, RVC_MATCH_C_BEQZ) -__RISCV_INSN_FUNCS(c_bnez, RVC_MASK_C_BNEZ, RVC_MATCH_C_BNEZ) -__RISCV_INSN_FUNCS(c_ebreak, RVC_MASK_C_EBREAK, RVC_MATCH_C_EBREAK) -__RISCV_INSN_FUNCS(ebreak, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) -__RISCV_INSN_FUNCS(sret, RVG_MASK_SRET, RVG_MATCH_SRET) -__RISCV_INSN_FUNCS(fence, RVG_MASK_FENCE, RVG_MATCH_FENCE); - /* special case to catch _any_ system instruction */ static __always_inline bool riscv_insn_is_system(u32 code) { @@ -275,248 +80,26 @@ static __always_inline bool riscv_insn_is_branch(u32 code) return (code & RV_INSN_OPCODE_MASK) == RVG_OPCODE_BRANCH; } -static __always_inline bool riscv_insn_is_c_jr(u32 code) -{ - return (code & RVC_MASK_C_JR) == RVC_MATCH_C_JR && - (code & RVC_INSN_J_RS1_MASK) != 0; -} - -static __always_inline bool riscv_insn_is_c_jalr(u32 code) -{ - return (code & RVC_MASK_C_JALR) == RVC_MATCH_C_JALR && - (code & RVC_INSN_J_RS1_MASK) != 0; -} - -#define INSN_MATCH_LB 0x3 -#define INSN_MASK_LB 0x707f -#define INSN_MATCH_LH 0x1003 -#define INSN_MASK_LH 0x707f -#define INSN_MATCH_LW 0x2003 -#define INSN_MASK_LW 0x707f -#define INSN_MATCH_LD 0x3003 -#define INSN_MASK_LD 0x707f -#define INSN_MATCH_LBU 0x4003 -#define INSN_MASK_LBU 0x707f -#define INSN_MATCH_LHU 0x5003 -#define INSN_MASK_LHU 0x707f -#define INSN_MATCH_LWU 0x6003 -#define INSN_MASK_LWU 0x707f -#define INSN_MATCH_SB 0x23 -#define INSN_MASK_SB 0x707f -#define INSN_MATCH_SH 0x1023 -#define INSN_MASK_SH 0x707f -#define INSN_MATCH_SW 0x2023 -#define INSN_MASK_SW 0x707f -#define INSN_MATCH_SD 0x3023 -#define INSN_MASK_SD 0x707f - -#define INSN_MATCH_C_LD 0x6000 -#define INSN_MASK_C_LD 0xe003 -#define INSN_MATCH_C_SD 0xe000 -#define INSN_MASK_C_SD 0xe003 -#define INSN_MATCH_C_LW 0x4000 -#define INSN_MASK_C_LW 0xe003 -#define INSN_MATCH_C_SW 0xc000 -#define INSN_MASK_C_SW 0xe003 -#define INSN_MATCH_C_LDSP 0x6002 -#define INSN_MASK_C_LDSP 0xe003 -#define INSN_MATCH_C_SDSP 0xe002 -#define INSN_MASK_C_SDSP 0xe003 -#define INSN_MATCH_C_LWSP 0x4002 -#define INSN_MASK_C_LWSP 0xe003 -#define INSN_MATCH_C_SWSP 0xc002 -#define INSN_MASK_C_SWSP 0xe003 - #define INSN_OPCODE_MASK 0x007c #define INSN_OPCODE_SHIFT 2 #define INSN_OPCODE_SYSTEM 28 -#define INSN_MASK_WFI 0xffffffff -#define INSN_MATCH_WFI 0x10500073 - -#define INSN_MASK_WRS 0xffffffff -#define INSN_MATCH_WRS 0x00d00073 - -#define INSN_MATCH_CSRRW 0x1073 -#define INSN_MASK_CSRRW 0x707f -#define INSN_MATCH_CSRRS 0x2073 -#define INSN_MASK_CSRRS 0x707f -#define INSN_MATCH_CSRRC 0x3073 -#define INSN_MASK_CSRRC 0x707f -#define INSN_MATCH_CSRRWI 0x5073 -#define INSN_MASK_CSRRWI 0x707f -#define INSN_MATCH_CSRRSI 0x6073 -#define INSN_MASK_CSRRSI 0x707f -#define INSN_MATCH_CSRRCI 0x7073 -#define INSN_MASK_CSRRCI 0x707f - -#define INSN_MATCH_FLW 0x2007 -#define INSN_MASK_FLW 0x707f -#define INSN_MATCH_FLD 0x3007 -#define INSN_MASK_FLD 0x707f -#define INSN_MATCH_FLQ 0x4007 -#define INSN_MASK_FLQ 0x707f -#define INSN_MATCH_FSW 0x2027 -#define INSN_MASK_FSW 0x707f -#define INSN_MATCH_FSD 0x3027 -#define INSN_MASK_FSD 0x707f -#define INSN_MATCH_FSQ 0x4027 -#define INSN_MASK_FSQ 0x707f - -#define INSN_MATCH_C_FLD 0x2000 -#define INSN_MASK_C_FLD 0xe003 -#define INSN_MATCH_C_FLW 0x6000 -#define INSN_MASK_C_FLW 0xe003 -#define INSN_MATCH_C_FSD 0xa000 -#define INSN_MASK_C_FSD 0xe003 -#define INSN_MATCH_C_FSW 0xe000 -#define INSN_MASK_C_FSW 0xe003 -#define INSN_MATCH_C_FLDSP 0x2002 -#define INSN_MASK_C_FLDSP 0xe003 -#define INSN_MATCH_C_FSDSP 0xa002 -#define INSN_MASK_C_FSDSP 0xe003 -#define INSN_MATCH_C_FLWSP 0x6002 -#define INSN_MASK_C_FLWSP 0xe003 -#define INSN_MATCH_C_FSWSP 0xe002 -#define INSN_MASK_C_FSWSP 0xe003 - -#define INSN_MATCH_C_LHU 0x8400 -#define INSN_MASK_C_LHU 0xfc43 -#define INSN_MATCH_C_LH 0x8440 -#define INSN_MASK_C_LH 0xfc43 -#define INSN_MATCH_C_SH 0x8c00 -#define INSN_MASK_C_SH 0xfc43 - #define INSN_16BIT_MASK 0x3 #define INSN_IS_16BIT(insn) (((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK) #define INSN_LEN(insn) (INSN_IS_16BIT(insn) ? 2 : 4) -#define SHIFT_RIGHT(x, y) \ - ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) - #define REG_MASK \ ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) -#define REG_OFFSET(insn, pos) \ - (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) - -#define REG_PTR(insn, pos, regs) \ - ((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))) - -#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) -#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) -#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) -#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) -#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) -#define GET_SP(regs) (*REG_PTR(2, 0, regs)) -#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) -#define IMM_I(insn) ((s32)(insn) >> 20) -#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ - (s32)(((insn) >> 7) & 0x1f)) - -#define SH_RD 7 -#define SH_RS1 15 -#define SH_RS2 20 -#define SH_RS2C 2 -#define MASK_RX 0x1f - #if defined(CONFIG_64BIT) #define LOG_REGBYTES 3 #else #define LOG_REGBYTES 2 #endif -#define MASK_FUNCT3 0x7000 - -#define GET_FUNCT3(insn) (((insn) >> 12) & 7) - -#define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) -#define RVC_IMM_SIGN(x) (-(((x) >> 12) & 1)) -#define RV_X_MASK(X, s, mask) (((X) >> (s)) & (mask)) -#define RV_X(X, s, n) RV_X_MASK(X, s, ((1 << (n)) - 1)) -#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ - (RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 5, 1) << 6)) -#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 5, 2) << 6)) -#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ - (RV_X(x, 12, 1) << 5) | \ - (RV_X(x, 2, 2) << 6)) -#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ - (RV_X(x, 12, 1) << 5) | \ - (RV_X(x, 2, 3) << 6)) -#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ - (RV_X(x, 7, 2) << 6)) -#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 7, 3) << 6)) -#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) -#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) -#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) -#define RVC_X(X, s, mask) RV_X_MASK(X, s, mask) - -#define RV_EXTRACT_FUNCT3(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RV_INSN_FUNCT3_OPOFF, \ - RV_INSN_FUNCT3_MASK >> RV_INSN_FUNCT3_OPOFF)); }) - -#define RV_EXTRACT_RS1_REG(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RVG_RS1_OPOFF, RVG_RS1_MASK)); }) - -#define RV_EXTRACT_RS2_REG(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RVG_RS2_OPOFF, RVG_RS2_MASK)); }) - -#define RV_EXTRACT_RD_REG(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RVG_RD_OPOFF, RVG_RD_MASK)); }) - -#define RV_EXTRACT_UTYPE_IMM(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RV_U_IMM_31_12_OPOFF, RV_U_IMM_31_12_MASK)); }) - -#define RV_EXTRACT_JTYPE_IMM(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RV_J_IMM_10_1_OPOFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OFF) | \ - (RV_X_MASK(x_, RV_J_IMM_11_OPOFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OFF) | \ - (RV_X_MASK(x_, RV_J_IMM_19_12_OPOFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OFF) | \ - (RV_IMM_SIGN(x_) << RV_J_IMM_SIGN_OFF); }) - -#define RV_EXTRACT_ITYPE_IMM(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RV_I_IMM_11_0_OPOFF, RV_I_IMM_11_0_MASK)) | \ - (RV_IMM_SIGN(x_) << RV_I_IMM_SIGN_OFF); }) - -#define RV_EXTRACT_BTYPE_IMM(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RV_B_IMM_4_1_OPOFF, RV_B_IMM_4_1_MASK) << RV_B_IMM_4_1_OFF) | \ - (RV_X_MASK(x_, RV_B_IMM_10_5_OPOFF, RV_B_IMM_10_5_MASK) << RV_B_IMM_10_5_OFF) | \ - (RV_X_MASK(x_, RV_B_IMM_11_OPOFF, RV_B_IMM_11_MASK) << RV_B_IMM_11_OFF) | \ - (RV_IMM_SIGN(x_) << RV_B_IMM_SIGN_OFF); }) - -#define RVC_EXTRACT_C2_RS1_REG(x) \ - ({typeof(x) x_ = (x); \ - (RV_X_MASK(x_, RVC_C2_RS1_OPOFF, RVC_C2_RS1_MASK)); }) - -#define RVC_EXTRACT_JTYPE_IMM(x) \ - ({typeof(x) x_ = (x); \ - (RVC_X(x_, RVC_J_IMM_3_1_OPOFF, RVC_J_IMM_3_1_MASK) << RVC_J_IMM_3_1_OFF) | \ - (RVC_X(x_, RVC_J_IMM_4_OPOFF, RVC_J_IMM_4_MASK) << RVC_J_IMM_4_OFF) | \ - (RVC_X(x_, RVC_J_IMM_5_OPOFF, RVC_J_IMM_5_MASK) << RVC_J_IMM_5_OFF) | \ - (RVC_X(x_, RVC_J_IMM_6_OPOFF, RVC_J_IMM_6_MASK) << RVC_J_IMM_6_OFF) | \ - (RVC_X(x_, RVC_J_IMM_7_OPOFF, RVC_J_IMM_7_MASK) << RVC_J_IMM_7_OFF) | \ - (RVC_X(x_, RVC_J_IMM_9_8_OPOFF, RVC_J_IMM_9_8_MASK) << RVC_J_IMM_9_8_OFF) | \ - (RVC_X(x_, RVC_J_IMM_10_OPOFF, RVC_J_IMM_10_MASK) << RVC_J_IMM_10_OFF) | \ - (RVC_IMM_SIGN(x_) << RVC_J_IMM_SIGN_OFF); }) - -#define RVC_EXTRACT_BTYPE_IMM(x) \ - ({typeof(x) x_ = (x); \ - (RVC_X(x_, RVC_B_IMM_2_1_OPOFF, RVC_B_IMM_2_1_MASK) << RVC_B_IMM_2_1_OFF) | \ - (RVC_X(x_, RVC_B_IMM_4_3_OPOFF, RVC_B_IMM_4_3_MASK) << RVC_B_IMM_4_3_OFF) | \ - (RVC_X(x_, RVC_B_IMM_5_OPOFF, RVC_B_IMM_5_MASK) << RVC_B_IMM_5_OFF) | \ - (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ - (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) +#define RV_X_MASK(X, s, mask) (((X) >> (s)) & (mask)) +// These three are used by vector stuff #define RVG_EXTRACT_SYSTEM_CSR(x) \ ({typeof(x) x_ = (x); RV_X_MASK(x_, RVG_SYSTEM_CSR_OFF, RVG_SYSTEM_CSR_MASK); }) @@ -526,78 +109,32 @@ static __always_inline bool riscv_insn_is_c_jalr(u32 code) #define RVV_EXTRACT_VL_VS_WIDTH(x) RVFDQ_EXTRACT_FL_FS_WIDTH(x) -/* - * Get the immediate from a J-type instruction. - * - * @insn: instruction to process - * Return: immediate - */ -static inline s32 riscv_insn_extract_jtype_imm(u32 insn) -{ - return RV_EXTRACT_JTYPE_IMM(insn); -} - -/* - * Update a J-type instruction with an immediate value. - * - * @insn: pointer to the jtype instruction - * @imm: the immediate to insert into the instruction - */ -static inline void riscv_insn_insert_jtype_imm(u32 *insn, s32 imm) +static inline unsigned long riscv_insn_reg_get_val(unsigned long *regs, u32 index) { - /* drop the old IMMs, all jal IMM bits sit at 31:12 */ - *insn &= ~GENMASK(31, 12); - *insn |= (RV_X_MASK(imm, RV_J_IMM_10_1_OFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OPOFF) | - (RV_X_MASK(imm, RV_J_IMM_11_OFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OPOFF) | - (RV_X_MASK(imm, RV_J_IMM_19_12_OFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OPOFF) | - (RV_X_MASK(imm, RV_J_IMM_SIGN_OFF, 1) << RV_J_IMM_SIGN_OPOFF); + /* register 0 is always 0 and not stored in the register struct */ + return index ? *(regs + index) : 0; } -/* - * Put together one immediate from a U-type and I-type instruction pair. - * - * The U-type contains an upper immediate, meaning bits[31:12] with [11:0] - * being zero, while the I-type contains a 12bit immediate. - * Combined these can encode larger 32bit values and are used for example - * in auipc + jalr pairs to allow larger jumps. - * - * @utype_insn: instruction containing the upper immediate - * @itype_insn: instruction - * Return: combined immediate - */ -static inline s32 riscv_insn_extract_utype_itype_imm(u32 utype_insn, u32 itype_insn) +static inline void riscv_insn_reg_set_val(unsigned long *regs, u32 index, unsigned long val) { - s32 imm; - - imm = RV_EXTRACT_UTYPE_IMM(utype_insn); - imm += RV_EXTRACT_ITYPE_IMM(itype_insn); - - return imm; + /* register 0 is always 0 and not stored in the register struct */ + if (index != 0) + *(regs + index) = val; } -/* - * Update a set of two instructions (U-type + I-type) with an immediate value. - * - * Used for example in auipc+jalrs pairs the U-type instructions contains - * a 20bit upper immediate representing bits[31:12], while the I-type - * instruction contains a 12bit immediate representing bits[11:0]. - * - * This also takes into account that both separate immediates are - * considered as signed values, so if the I-type immediate becomes - * negative (BIT(11) set) the U-type part gets adjusted. - * - * @utype_insn: pointer to the utype instruction of the pair - * @itype_insn: pointer to the itype instruction of the pair - * @imm: the immediate to insert into the two instructions - */ -static inline void riscv_insn_insert_utype_itype_imm(u32 *utype_insn, u32 *itype_insn, s32 imm) -{ - /* drop possible old IMM values */ - *utype_insn &= ~(RV_U_IMM_31_12_MASK); - *itype_insn &= ~(RV_I_IMM_11_0_MASK << RV_I_IMM_11_0_OPOFF); +#define riscv_insn_branch(_insn, regs_ptr, _opcode, _pc, _comparison, type) \ + ({ \ + unsigned long _ret; \ + if ((type)riscv_insn_reg_get_val( \ + regs_ptr, \ + riscv_insn_##_insn##_extract_xs1(_opcode)) \ + _comparison(type) riscv_insn_reg_get_val( \ + regs_ptr, \ + riscv_insn_##_insn##_extract_xs2(_opcode))) \ + _ret = riscv_insn_##_insn##_extract_imm(_opcode); \ + else \ + _ret = _pc + 4; \ + _ret; \ + }) - /* add the adapted IMMs */ - *utype_insn |= (imm & RV_U_IMM_31_12_MASK) + ((imm & BIT(11)) << 1); - *itype_insn |= ((imm & RV_I_IMM_11_0_MASK) << RV_I_IMM_11_0_OPOFF); -} #endif /* _ASM_RISCV_INSN_H */ diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 24585304c02b14..709a1c18ce227f 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -183,6 +183,12 @@ struct kvm_vcpu_reset_state { unsigned long a1; }; +#ifdef CONFIG_RISCV_KVM_TEST_CSR +struct kvm_test_csr { + unsigned long val; +}; +#endif + struct kvm_vcpu_arch { /* VCPU ran at least once */ bool ran_atleast_once; @@ -278,6 +284,10 @@ struct kvm_vcpu_arch { gpa_t shmem; u64 last_steal; } sta; + +#ifdef CONFIG_RISCV_KVM_TEST_CSR + struct kvm_test_csr test_csr; +#endif }; /* diff --git a/arch/riscv/include/asm/kvm_vcpu_insn.h b/arch/riscv/include/asm/kvm_vcpu_insn.h index 350011c83581cb..01efdaaede217b 100644 --- a/arch/riscv/include/asm/kvm_vcpu_insn.h +++ b/arch/riscv/include/asm/kvm_vcpu_insn.h @@ -11,7 +11,7 @@ struct kvm_run; struct kvm_cpu_trap; struct kvm_mmio_decode { - unsigned long insn; + unsigned long rd; int insn_len; int len; int shift; @@ -19,7 +19,8 @@ struct kvm_mmio_decode { }; struct kvm_csr_decode { - unsigned long insn; + unsigned long rd; + unsigned long insn_len; int return_handled; }; diff --git a/arch/riscv/include/asm/kvm_vcpu_test_csr.h b/arch/riscv/include/asm/kvm_vcpu_test_csr.h new file mode 100644 index 00000000000000..a844fccaafc346 --- /dev/null +++ b/arch/riscv/include/asm/kvm_vcpu_test_csr.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __KVM_VCPU_RISCV_TEST_CSR_H +#define __KVM_VCPU_RISCV_TEST_CSR_H + +#include + +#define KVM_RISCV_VCPU_TEST_CSR_FUNCS \ + {.base = CSR_VSSCRATCH, .count = 1, .func = kvm_riscv_vcpu_test_csr }, + +int kvm_riscv_vcpu_test_csr(struct kvm_vcpu *vcpu, unsigned int csr_num, + unsigned long *val, unsigned long new_val, + unsigned long wr_mask); + +#endif /* !__KVM_VCPU_RISCV_TEST_CSR_H */ diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 7642704c7f1841..b26a90eb65ccac 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -78,14 +79,24 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, u32 jalr_insn, int patch_offset) { u32 call[2] = { auipc_insn, jalr_insn }; + u32 auipc_imm; s32 imm; /* get and adjust new target address */ - imm = riscv_insn_extract_utype_itype_imm(auipc_insn, jalr_insn); + imm = riscv_insn_auipc_extract_imm(auipc_insn) + riscv_insn_jalr_extract_imm(jalr_insn); imm -= patch_offset; + /* + * When the 32-bit immediate is split across auipc and jalr, the + * constructed immediates need to be treated as individually sign + * extended numbers. Add the sign bit of the lower 12 bits to the upper + * 20 bits to undo the bleeding of the sign. + */ + auipc_imm = imm + (BIT(11) << 1); + /* update instructions */ - riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm); + riscv_insn_auipc_insert_imm(&call[0], auipc_imm); + riscv_insn_jalr_insert_imm(&call[1], imm); /* patch the call place again */ patch_text_nosync(ptr, call, sizeof(u32) * 2); @@ -96,11 +107,11 @@ static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) s32 imm; /* get and adjust new target address */ - imm = riscv_insn_extract_jtype_imm(jal_insn); + imm = riscv_insn_jal_extract_imm(jal_insn); imm -= patch_offset; /* update instruction */ - riscv_insn_insert_jtype_imm(&jal_insn, imm); + riscv_insn_jal_insert_imm(&jal_insn, imm); /* patch the call place again */ patch_text_nosync(ptr, &jal_insn, sizeof(u32)); @@ -127,7 +138,7 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, continue; /* if instruction pair is a call, it will use the ra register */ - if (RV_EXTRACT_RD_REG(insn) != 1) + if (riscv_insn_jalr_extract_xd(insn) != 1) continue; riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), @@ -136,7 +147,7 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, } if (riscv_insn_is_jal(insn)) { - s32 imm = riscv_insn_extract_jtype_imm(insn); + s32 imm = riscv_insn_jal_extract_imm(insn); /* Don't modify jumps inside the alternative block */ if ((alt_ptr + i * sizeof(u32) + imm) >= alt_ptr && diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c index 6ec9dbd7292eec..e38d5f863747af 100644 --- a/arch/riscv/kernel/cfi.c +++ b/arch/riscv/kernel/cfi.c @@ -40,16 +40,16 @@ static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target, if (!riscv_insn_is_beq(insn)) return false; - *type = (u32)regs_ptr[RV_EXTRACT_RS1_REG(insn)]; + *type = (u32)regs_ptr[riscv_insn_beq_extract_xs1(insn)]; if (get_kernel_nofault(insn, (void *)regs->epc) || get_kernel_nofault(insn, (void *)regs->epc + GET_INSN_LENGTH(insn))) return false; if (riscv_insn_is_jalr(insn)) - rs1_num = RV_EXTRACT_RS1_REG(insn); + rs1_num = riscv_insn_jalr_extract_xs1(insn); else if (riscv_insn_is_c_jalr(insn)) - rs1_num = RVC_EXTRACT_C2_RS1_REG(insn); + rs1_num = riscv_insn_c_jalr_extract_xs1(insn); else return false; diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 0bf629204c76a4..fcf7cda3cc33fb 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -23,97 +23,73 @@ enum { static unsigned long stepped_address; static unsigned int stepped_opcode; -static int decode_register_index(unsigned long opcode, int offset) -{ - return (opcode >> offset) & 0x1F; -} - -static int decode_register_index_short(unsigned long opcode, int offset) -{ - return ((opcode >> offset) & 0x7) + 8; -} - -/* Calculate the new address for after a step */ static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) { unsigned long pc = regs->epc; unsigned long *regs_ptr = (unsigned long *)regs; - unsigned int rs1_num, rs2_num; + unsigned int rs1_num; int op_code; if (get_kernel_nofault(op_code, (void *)pc)) return -EINVAL; + if ((op_code & __INSN_LENGTH_MASK) != __INSN_LENGTH_GE_32) { - if (riscv_insn_is_c_jalr(op_code) || - riscv_insn_is_c_jr(op_code)) { - rs1_num = decode_register_index(op_code, RVC_C2_RS1_OPOFF); - *next_addr = regs_ptr[rs1_num]; - } else if (riscv_insn_is_c_j(op_code) || - riscv_insn_is_c_jal(op_code)) { - *next_addr = RVC_EXTRACT_JTYPE_IMM(op_code) + pc; + if (riscv_insn_is_c_jalr(op_code)) { + *next_addr = regs_ptr[riscv_insn_c_jalr_extract_xs1(op_code)]; + } else if (riscv_insn_is_c_jr(op_code)) { + *next_addr = regs_ptr[riscv_insn_c_jr_extract_xs1(op_code)]; + } else if (riscv_insn_is_c_j(op_code)) { + *next_addr = riscv_insn_c_j_extract_imm(op_code) + pc; + } else if (riscv_insn_is_c_jal(op_code)) { + *next_addr = riscv_insn_c_jal_extract_imm(op_code) + pc; } else if (riscv_insn_is_c_beqz(op_code)) { - rs1_num = decode_register_index_short(op_code, - RVC_C1_RS1_OPOFF); - if (!rs1_num || regs_ptr[rs1_num] == 0) - *next_addr = RVC_EXTRACT_BTYPE_IMM(op_code) + pc; + rs1_num = riscv_insn_c_beqz_extract_xs1(op_code); + if (regs_ptr[8 + rs1_num] == 0) + *next_addr = riscv_insn_c_beqz_extract_imm(op_code) + pc; else *next_addr = pc + 2; } else if (riscv_insn_is_c_bnez(op_code)) { - rs1_num = - decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); - if (rs1_num && regs_ptr[rs1_num] != 0) - *next_addr = RVC_EXTRACT_BTYPE_IMM(op_code) + pc; + rs1_num = riscv_insn_c_bnez_extract_xs1(op_code); + if (regs_ptr[8 + rs1_num] != 0) + *next_addr = riscv_insn_c_bnez_extract_imm(op_code) + pc; else *next_addr = pc + 2; } else { *next_addr = pc + 2; } } else { - if ((op_code & __INSN_OPCODE_MASK) == __INSN_BRANCH_OPCODE) { - bool result = false; - long imm = RV_EXTRACT_BTYPE_IMM(op_code); - unsigned long rs1_val = 0, rs2_val = 0; - - rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); - rs2_num = decode_register_index(op_code, RVG_RS2_OPOFF); - if (rs1_num) - rs1_val = regs_ptr[rs1_num]; - if (rs2_num) - rs2_val = regs_ptr[rs2_num]; - - if (riscv_insn_is_beq(op_code)) - result = (rs1_val == rs2_val) ? true : false; - else if (riscv_insn_is_bne(op_code)) - result = (rs1_val != rs2_val) ? true : false; - else if (riscv_insn_is_blt(op_code)) - result = - ((long)rs1_val < - (long)rs2_val) ? true : false; - else if (riscv_insn_is_bge(op_code)) - result = - ((long)rs1_val >= - (long)rs2_val) ? true : false; - else if (riscv_insn_is_bltu(op_code)) - result = (rs1_val < rs2_val) ? true : false; - else if (riscv_insn_is_bgeu(op_code)) - result = (rs1_val >= rs2_val) ? true : false; - if (result) - *next_addr = imm + pc; - else - *next_addr = pc + 4; + if (riscv_insn_is_beq(op_code)) { + *next_addr = riscv_insn_branch(beq, regs_ptr, op_code, + pc, ==, unsigned long); + } else if (riscv_insn_is_bne(op_code)) { + *next_addr = riscv_insn_branch(bne, regs_ptr, op_code, + pc, !=, unsigned long); + } else if (riscv_insn_is_blt(op_code)) { + *next_addr = riscv_insn_branch(blt, regs_ptr, op_code, + pc, <, long); + } else if (riscv_insn_is_bge(op_code)) { + *next_addr = riscv_insn_branch(bge, regs_ptr, op_code, + pc, >=, long); + } else if (riscv_insn_is_bltu(op_code)) { + *next_addr = riscv_insn_branch(bltu, regs_ptr, op_code, + pc, <, unsigned long); + } else if (riscv_insn_is_bgeu(op_code)) { + *next_addr = riscv_insn_branch(bgeu, regs_ptr, op_code, + pc, >=, unsigned long); } else if (riscv_insn_is_jal(op_code)) { - *next_addr = RV_EXTRACT_JTYPE_IMM(op_code) + pc; + *next_addr = riscv_insn_jal_extract_imm(op_code) + pc; } else if (riscv_insn_is_jalr(op_code)) { - rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); + rs1_num = riscv_insn_jalr_extract_xs1(op_code); if (rs1_num) - *next_addr = ((unsigned long *)regs)[rs1_num]; - *next_addr += RV_EXTRACT_ITYPE_IMM(op_code); + *next_addr = regs_ptr[rs1_num]; + *next_addr += riscv_insn_jalr_extract_imm(op_code); } else if (riscv_insn_is_sret(op_code)) { *next_addr = pc; } else { *next_addr = pc + 4; } } + return 0; } diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index 54e2d9552e930d..cad40e4afd8678 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -116,32 +116,6 @@ static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1)) #define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x)) -#define ENCODE_ITYPE_IMM(x) \ - (RV_X(x, 0, 12) << 20) -#define ENCODE_BTYPE_IMM(x) \ - ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \ - (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31)) -#define ENCODE_UTYPE_IMM(x) \ - (RV_X(x, 12, 20) << 12) -#define ENCODE_JTYPE_IMM(x) \ - ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \ - (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31)) -#define ENCODE_CBTYPE_IMM(x) \ - ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \ - (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12)) -#define ENCODE_CJTYPE_IMM(x) \ - ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \ - (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \ - (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12)) -#define ENCODE_UJTYPE_IMM(x) \ - (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \ - (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32)) -#define ENCODE_UITYPE_IMM(x) \ - (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32)) - -#define CLEAN_IMM(type, x) \ - ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x)) - int arch_kexec_apply_relocations_add(struct purgatory_info *pi, Elf_Shdr *section, const Elf_Shdr *relsec, @@ -197,12 +171,14 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, switch (r_type) { case R_RISCV_BRANCH: - *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) | - ENCODE_BTYPE_IMM(val - addr); + /* + * For simplicity, use beq as represenative of all + * branches (they all have the same imm encoding) + */ + riscv_insn_beq_insert_imm((u32 *)loc, val - addr); break; case R_RISCV_JAL: - *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) | - ENCODE_JTYPE_IMM(val - addr); + riscv_insn_jal_insert_imm((u32 *)loc, val - addr); break; /* * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I @@ -213,16 +189,23 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, case R_RISCV_PCREL_HI20: case R_RISCV_CALL_PLT: case R_RISCV_CALL: - *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) | - ENCODE_UJTYPE_IMM(val - addr); + riscv_insn_auipc_insert_imm((u32 *)loc, RISCV_CONST_HIGH_PART(val - addr)); + riscv_insn_jalr_insert_imm((u32 *)loc + 1, + RISCV_CONST_LOW_PART(val - addr)); break; case R_RISCV_RVC_BRANCH: - *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) | - ENCODE_CBTYPE_IMM(val - addr); + /* + * For simplicity, use c.beqz as represenative of all + * compressed branches (they all have the same imm encoding) + */ + riscv_insn_c_beqz_insert_imm((u16 *)loc, val - addr); break; case R_RISCV_RVC_JUMP: - *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) | - ENCODE_CJTYPE_IMM(val - addr); + /* + * For simplicity, use c.j as represenative of all + * compressed jumps (they all have the same imm encoding) + */ + riscv_insn_c_j_insert_imm((u16 *)loc, val - addr); break; case R_RISCV_ADD16: *(u16 *)loc += val; diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c index 65d9590bfb9ff5..0d70c8301a451f 100644 --- a/arch/riscv/kernel/probes/decode-insn.c +++ b/arch/riscv/kernel/probes/decode-insn.c @@ -42,7 +42,12 @@ riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) RISCV_INSN_SET_SIMULATE(jal, insn); RISCV_INSN_SET_SIMULATE(jalr, insn); RISCV_INSN_SET_SIMULATE(auipc, insn); - RISCV_INSN_SET_SIMULATE(branch, insn); + RISCV_INSN_SET_SIMULATE(beq, insn); + RISCV_INSN_SET_SIMULATE(bne, insn); + RISCV_INSN_SET_SIMULATE(blt, insn); + RISCV_INSN_SET_SIMULATE(bge, insn); + RISCV_INSN_SET_SIMULATE(bltu, insn); + RISCV_INSN_SET_SIMULATE(bgeu, insn); return INSN_GOOD; } diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c index fa581590c1f8b2..d086d3c6474c53 100644 --- a/arch/riscv/kernel/probes/simulate-insn.c +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -4,236 +4,163 @@ #include #include -#include "decode-insn.h" +#include #include "simulate-insn.h" -static inline bool rv_insn_reg_get_val(struct pt_regs *regs, u32 index, - unsigned long *ptr) -{ - if (index == 0) - *ptr = 0; - else if (index <= 31) - *ptr = *((unsigned long *)regs + index); - else - return false; - - return true; -} - -static inline bool rv_insn_reg_set_val(struct pt_regs *regs, u32 index, - unsigned long val) -{ - if (index == 0) - return true; - else if (index <= 31) - *((unsigned long *)regs + index) = val; - else - return false; - - return true; -} - bool __kprobes simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * 31 30 21 20 19 12 11 7 6 0 - * imm [20] | imm[10:1] | imm[11] | imm[19:12] | rd | opcode - * 1 10 1 8 5 JAL/J - */ - bool ret; - s32 imm; - u32 index = RV_EXTRACT_RD_REG(opcode); + s32 imm = riscv_insn_jal_extract_imm(opcode); + u32 index = riscv_insn_jal_extract_xd(opcode); - ret = rv_insn_reg_set_val(regs, index, addr + 4); - if (!ret) - return ret; - - imm = RV_EXTRACT_JTYPE_IMM(opcode); + riscv_insn_reg_set_val((unsigned long *)regs, index, addr + 4); instruction_pointer_set(regs, addr + imm); - return ret; + return true; } bool __kprobes simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * 31 20 19 15 14 12 11 7 6 0 - * offset[11:0] | rs1 | 010 | rd | opcode - * 12 5 3 5 JALR/JR - */ - bool ret; unsigned long base_addr; - u32 imm = RV_EXTRACT_ITYPE_IMM(opcode); - u32 rd_index = RV_EXTRACT_RD_REG(opcode); - u32 rs1_index = RV_EXTRACT_RS1_REG(opcode); + s32 imm = riscv_insn_jalr_extract_imm(opcode); + u32 rd_index = riscv_insn_jalr_extract_xd(opcode); + u32 rs1_index = riscv_insn_jalr_extract_xs1(opcode); - ret = rv_insn_reg_get_val(regs, rs1_index, &base_addr); - if (!ret) - return ret; + base_addr = riscv_insn_reg_get_val((unsigned long *)regs, rs1_index); - ret = rv_insn_reg_set_val(regs, rd_index, addr + 4); - if (!ret) - return ret; + riscv_insn_reg_set_val((unsigned long *)regs, rd_index, addr + 4); - instruction_pointer_set(regs, (base_addr + sign_extend32((imm), 11))&~1); + instruction_pointer_set(regs, (base_addr + imm) & ~1); - return ret; + return true; } bool __kprobes simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * auipc instruction: - * 31 12 11 7 6 0 - * | imm[31:12] | rd | opcode | - * 20 5 7 - */ + u32 rd_index = riscv_insn_auipc_extract_xd(opcode); + unsigned long rd_val = addr + (s32)riscv_insn_auipc_extract_imm(opcode); - u32 rd_idx = RV_EXTRACT_RD_REG(opcode); - unsigned long rd_val = addr + (s32)RV_EXTRACT_UTYPE_IMM(opcode); - - if (!rv_insn_reg_set_val(regs, rd_idx, rd_val)) - return false; + riscv_insn_reg_set_val((unsigned long *)regs, rd_index, rd_val); instruction_pointer_set(regs, addr + 4); - return true; } -bool __kprobes simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs) +bool __kprobes simulate_beq(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * branch instructions: - * 31 30 25 24 20 19 15 14 12 11 8 7 6 0 - * | imm[12] | imm[10:5] | rs2 | rs1 | funct3 | imm[4:1] | imm[11] | opcode | - * 1 6 5 5 3 4 1 7 - * imm[12|10:5] rs2 rs1 000 imm[4:1|11] 1100011 BEQ - * imm[12|10:5] rs2 rs1 001 imm[4:1|11] 1100011 BNE - * imm[12|10:5] rs2 rs1 100 imm[4:1|11] 1100011 BLT - * imm[12|10:5] rs2 rs1 101 imm[4:1|11] 1100011 BGE - * imm[12|10:5] rs2 rs1 110 imm[4:1|11] 1100011 BLTU - * imm[12|10:5] rs2 rs1 111 imm[4:1|11] 1100011 BGEU - */ - - s32 offset; - s32 offset_tmp; - unsigned long rs1_val; - unsigned long rs2_val; - - if (!rv_insn_reg_get_val(regs, RV_EXTRACT_RS1_REG(opcode), &rs1_val) || - !rv_insn_reg_get_val(regs, RV_EXTRACT_RS2_REG(opcode), &rs2_val)) - return false; - - offset_tmp = RV_EXTRACT_BTYPE_IMM(opcode); - switch (RV_EXTRACT_FUNCT3(opcode)) { - case RVG_FUNCT3_BEQ: - offset = (rs1_val == rs2_val) ? offset_tmp : 4; - break; - case RVG_FUNCT3_BNE: - offset = (rs1_val != rs2_val) ? offset_tmp : 4; - break; - case RVG_FUNCT3_BLT: - offset = ((long)rs1_val < (long)rs2_val) ? offset_tmp : 4; - break; - case RVG_FUNCT3_BGE: - offset = ((long)rs1_val >= (long)rs2_val) ? offset_tmp : 4; - break; - case RVG_FUNCT3_BLTU: - offset = (rs1_val < rs2_val) ? offset_tmp : 4; - break; - case RVG_FUNCT3_BGEU: - offset = (rs1_val >= rs2_val) ? offset_tmp : 4; - break; - default: - return false; - } - - instruction_pointer_set(regs, addr + offset); + unsigned long next_addr; + next_addr = riscv_insn_branch(beq, (unsigned long *)regs, opcode, addr, ==, unsigned long); + instruction_pointer_set(regs, next_addr); return true; } -bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs) +bool __kprobes simulate_bne(u32 opcode, unsigned long addr, struct pt_regs *regs) { - s32 offset = RVC_EXTRACT_JTYPE_IMM(opcode); + unsigned long next_addr; - instruction_pointer_set(regs, addr + offset); + next_addr = riscv_insn_branch(bne, (unsigned long *)regs, opcode, addr, !=, unsigned long); + instruction_pointer_set(regs, next_addr); + return true; +} +bool __kprobes simulate_blt(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + unsigned long next_addr; + + next_addr = riscv_insn_branch(blt, (unsigned long *)regs, opcode, addr, <, long); + instruction_pointer_set(regs, next_addr); return true; } -static bool __kprobes simulate_c_jr_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs, - bool is_jalr) +bool __kprobes simulate_bge(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * 15 12 11 7 6 2 1 0 - * | funct4 | rs1 | rs2 | op | - * 4 5 5 2 - */ + unsigned long next_addr; - unsigned long jump_addr; + next_addr = riscv_insn_branch(bge, (unsigned long *)regs, opcode, addr, >=, long); + instruction_pointer_set(regs, next_addr); + return true; +} - u32 rs1 = RVC_EXTRACT_C2_RS1_REG(opcode); +bool __kprobes simulate_bltu(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + unsigned long next_addr; - if (rs1 == 0) /* C.JR is only valid when rs1 != x0 */ - return false; + next_addr = riscv_insn_branch(bltu, (unsigned long *)regs, opcode, addr, <, unsigned long); + instruction_pointer_set(regs, next_addr); + return true; +} - if (!rv_insn_reg_get_val(regs, rs1, &jump_addr)) - return false; +bool __kprobes simulate_bgeu(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + unsigned long next_addr; + + next_addr = riscv_insn_branch(bgeu, (unsigned long *)regs, opcode, addr, >=, unsigned long); + instruction_pointer_set(regs, next_addr); + return true; +} - if (is_jalr && !rv_insn_reg_set_val(regs, 1, addr + 2)) - return false; +bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + s32 offset = riscv_insn_c_j_extract_imm(opcode); - instruction_pointer_set(regs, jump_addr); + instruction_pointer_set(regs, addr + offset); return true; } bool __kprobes simulate_c_jr(u32 opcode, unsigned long addr, struct pt_regs *regs) { - return simulate_c_jr_jalr(opcode, addr, regs, false); + unsigned long next_addr; + unsigned long *regs_ptr = (unsigned long *)regs; + + next_addr = regs_ptr[riscv_insn_c_jr_extract_xs1(opcode)]; + instruction_pointer_set(regs, next_addr); + + regs->ra = addr + 2; + return true; } bool __kprobes simulate_c_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs) { - return simulate_c_jr_jalr(opcode, addr, regs, true); + unsigned long next_addr; + unsigned long *regs_ptr = (unsigned long *)regs; + + next_addr = regs_ptr[riscv_insn_c_jalr_extract_xs1(opcode)]; + instruction_pointer_set(regs, next_addr); + + regs->ra = addr + 2; + return true; } -static bool __kprobes simulate_c_bnez_beqz(u32 opcode, unsigned long addr, struct pt_regs *regs, - bool is_bnez) +bool __kprobes simulate_c_bnez(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * 15 13 12 10 9 7 6 2 1 0 - * | funct3 | offset[8|4:3] | rs1' | offset[7:6|2:1|5] | op | - * 3 3 3 5 2 - */ - - s32 offset; u32 rs1; - unsigned long rs1_val; + unsigned long offset; + unsigned long *regs_ptr = (unsigned long *)regs; - rs1 = 0x8 | ((opcode >> 7) & 0x7); - - if (!rv_insn_reg_get_val(regs, rs1, &rs1_val)) - return false; - - if ((rs1_val != 0 && is_bnez) || (rs1_val == 0 && !is_bnez)) - offset = RVC_EXTRACT_BTYPE_IMM(opcode); + rs1 = riscv_insn_c_bnez_extract_xs1(opcode); + if (regs_ptr[8 + rs1] != 0) + offset = riscv_insn_c_bnez_extract_imm(opcode); else offset = 2; instruction_pointer_set(regs, addr + offset); - return true; } -bool __kprobes simulate_c_bnez(u32 opcode, unsigned long addr, struct pt_regs *regs) -{ - return simulate_c_bnez_beqz(opcode, addr, regs, true); -} - bool __kprobes simulate_c_beqz(u32 opcode, unsigned long addr, struct pt_regs *regs) { - return simulate_c_bnez_beqz(opcode, addr, regs, false); + u32 rs1; + unsigned long offset; + unsigned long *regs_ptr = (unsigned long *)regs; + + rs1 = riscv_insn_c_beqz_extract_xs1(opcode); + if (regs_ptr[8 + rs1] == 0) + offset = riscv_insn_c_beqz_extract_imm(opcode); + else + offset = 2; + + instruction_pointer_set(regs, addr + offset); + return true; } diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index 44ebbc444db9da..f2f707e92dee3e 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -21,7 +21,12 @@ } while (0) bool simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *regs); -bool simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_beq(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_bne(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_blt(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_bge(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_bltu(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_bgeu(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs); diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c index 2a27d3ff4ac66d..a36ea0994ae862 100644 --- a/arch/riscv/kernel/traps_misaligned.c +++ b/arch/riscv/kernel/traps_misaligned.c @@ -22,15 +22,11 @@ #ifdef CONFIG_FPU -#define FP_GET_RD(insn) (insn >> 7 & 0x1F) - extern void put_f32_reg(unsigned long fp_reg, unsigned long value); -static int set_f32_rd(unsigned long insn, struct pt_regs *regs, +static int set_f32_rd(unsigned long fp_reg, struct pt_regs *regs, unsigned long val) { - unsigned long fp_reg = FP_GET_RD(insn); - put_f32_reg(fp_reg, val); regs->status |= SR_FS_DIRTY; @@ -39,9 +35,8 @@ static int set_f32_rd(unsigned long insn, struct pt_regs *regs, extern void put_f64_reg(unsigned long fp_reg, unsigned long value); -static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) +static int set_f64_rd(unsigned long fp_reg, struct pt_regs *regs, u64 val) { - unsigned long fp_reg = FP_GET_RD(insn); unsigned long value; #if __riscv_xlen == 32 @@ -58,10 +53,8 @@ static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) #if __riscv_xlen == 32 extern void get_f64_reg(unsigned long fp_reg, u64 *value); -static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, - struct pt_regs *regs) +static u64 get_f64_rs(unsigned long fp_reg, struct pt_regs *regs) { - unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; u64 val; get_f64_reg(fp_reg, &val); @@ -73,10 +66,8 @@ static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, extern unsigned long get_f64_reg(unsigned long fp_reg); -static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, - struct pt_regs *regs) +static unsigned long get_f64_rs(unsigned long fp_reg, struct pt_regs *regs) { - unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; unsigned long val; val = get_f64_reg(fp_reg); @@ -89,10 +80,8 @@ static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, extern unsigned long get_f32_reg(unsigned long fp_reg); -static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, - struct pt_regs *regs) +static unsigned long get_f32_rs(unsigned long fp_reg, struct pt_regs *regs) { - unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; unsigned long val; val = get_f32_reg(fp_reg); @@ -107,28 +96,18 @@ static void set_f32_rd(unsigned long insn, struct pt_regs *regs, static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {} -static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, - struct pt_regs *regs) +static unsigned long get_f64_rs(unsigned long fp_reg, struct pt_regs *regs) { return 0; } -static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, - struct pt_regs *regs) +static unsigned long get_f32_rs(unsigned long fp_reg, struct pt_regs *regs) { return 0; } #endif -#define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs)) -#define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs)) -#define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs)) - -#define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs)) -#define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs)) -#define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs)) - #define __read_insn(regs, insn, insn_addr, type) \ ({ \ int __ret; \ @@ -217,13 +196,13 @@ static int handle_vector_misaligned_load(struct pt_regs *regs) } #endif -static int handle_scalar_misaligned_load(struct pt_regs *regs) +static noinline int handle_scalar_misaligned_load(struct pt_regs *regs) { union reg_data val; unsigned long epc = regs->epc; unsigned long insn; unsigned long addr = regs->badaddr; - int fp = 0, shift = 0, len = 0; + int fp = 0, shift = 0, len = 0, rd = 0; perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); @@ -240,68 +219,71 @@ static int handle_scalar_misaligned_load(struct pt_regs *regs) regs->epc = 0; - if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { + if (riscv_insn_is_lw(insn)) { len = 4; shift = 8 * (sizeof(unsigned long) - len); -#if defined(CONFIG_64BIT) - } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { + rd = riscv_insn_lw_extract_xd(insn); + } else if (riscv_insn_is_ld(insn)) { len = 8; shift = 8 * (sizeof(unsigned long) - len); - } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { + rd = riscv_insn_ld_extract_xd(insn); + } else if (riscv_insn_is_lwu(insn)) { len = 4; -#endif - } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { + rd = riscv_insn_lwu_extract_xd(insn); + } else if (riscv_insn_is_fld(insn)) { fp = 1; len = 8; - } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { + rd = riscv_insn_fld_extract_fd(insn); + } else if (riscv_insn_is_flw(insn)) { fp = 1; len = 4; - } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { + rd = riscv_insn_flw_extract_fd(insn); + } else if (riscv_insn_is_lh(insn)) { len = 2; shift = 8 * (sizeof(unsigned long) - len); - } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { + rd = riscv_insn_lh_extract_xd(insn); + } else if (riscv_insn_is_lhu(insn)) { len = 2; -#if defined(CONFIG_64BIT) - } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { + rd = riscv_insn_lhu_extract_xd(insn); + } else if (riscv_insn_is_c_ld(insn)) { len = 8; shift = 8 * (sizeof(unsigned long) - len); - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && - ((insn >> SH_RD) & 0x1f)) { + rd = (8 + riscv_insn_c_ld_extract_xd(insn)); + } else if (riscv_insn_is_c_ldsp(insn)) { len = 8; shift = 8 * (sizeof(unsigned long) - len); -#endif - } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { + rd = riscv_insn_c_ldsp_extract_xd(insn); + } else if (riscv_insn_is_c_lw(insn)) { len = 4; shift = 8 * (sizeof(unsigned long) - len); - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && - ((insn >> SH_RD) & 0x1f)) { + rd = (8 + riscv_insn_c_lw_extract_xd(insn)); + } else if (riscv_insn_is_c_lwsp(insn)) { len = 4; shift = 8 * (sizeof(unsigned long) - len); - } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { + rd = riscv_insn_c_lwsp_extract_xd(insn); + } else if (riscv_insn_is_c_fld(insn)) { fp = 1; len = 8; - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { + rd = (8 + riscv_insn_c_fld_extract_fd(insn)); + } else if (riscv_insn_is_c_fldsp(insn)) { fp = 1; len = 8; -#if defined(CONFIG_32BIT) - } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { + rd = riscv_insn_c_fldsp_extract_fd(insn); + } else if (riscv_insn_is_c_flw(insn)) { fp = 1; len = 4; - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { + rd = (8 + riscv_insn_c_flw_extract_fd(insn)); + } else if (riscv_insn_is_c_flwsp(insn)) { fp = 1; len = 4; -#endif - } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) { + rd = riscv_insn_c_flwsp_extract_fd(insn); + } else if (riscv_insn_is_c_lhu(insn)) { len = 2; - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) { + rd = (8 + riscv_insn_c_lhu_extract_xd(insn)); + } else if (riscv_insn_is_c_lh(insn)) { len = 2; - shift = 8 * (sizeof(ulong) - len); - insn = RVC_RS2S(insn) << SH_RD; + shift = 8 * (sizeof(unsigned long) - len); + rd = (8 + riscv_insn_c_lh_extract_xd(insn)); } else { regs->epc = epc; return -1; @@ -319,11 +301,11 @@ static int handle_scalar_misaligned_load(struct pt_regs *regs) } if (!fp) - SET_RD(insn, regs, (long)(val.data_ulong << shift) >> shift); + *(unsigned long *)((unsigned long *)regs + rd) = val.data_ulong << shift; else if (len == 8) - set_f64_rd(insn, regs, val.data_u64); + set_f64_rd(rd, regs, val.data_u64); else - set_f32_rd(insn, regs, val.data_ulong); + set_f32_rd(rd, regs, val.data_ulong); regs->epc = epc + INSN_LEN(insn); @@ -336,7 +318,7 @@ static int handle_scalar_misaligned_store(struct pt_regs *regs) unsigned long epc = regs->epc; unsigned long insn; unsigned long addr = regs->badaddr; - int len = 0, fp = 0; + int fp = 0, len = 0, rd = 0; perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); @@ -351,67 +333,68 @@ static int handle_scalar_misaligned_store(struct pt_regs *regs) regs->epc = 0; - val.data_ulong = GET_RS2(insn, regs); - - if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { + if (riscv_insn_is_sw(insn)) { len = 4; -#if defined(CONFIG_64BIT) - } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { + rd = riscv_insn_sw_extract_xs2(insn); + } else if (riscv_insn_is_sd(insn)) { len = 8; -#endif - } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { + rd = riscv_insn_sd_extract_xs2(insn); + } else if (riscv_insn_is_fsd(insn)) { fp = 1; len = 8; - val.data_u64 = GET_F64_RS2(insn, regs); - } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { + rd = riscv_insn_fsd_extract_fs2(insn); + } else if (riscv_insn_is_fsw(insn)) { fp = 1; len = 4; - val.data_ulong = GET_F32_RS2(insn, regs); - } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { + rd = riscv_insn_fsw_extract_fs2(insn); + } else if (riscv_insn_is_sh(insn)) { len = 2; -#if defined(CONFIG_64BIT) - } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { + rd = riscv_insn_sh_extract_xs2(insn); + } else if (riscv_insn_is_c_sd(insn)) { len = 8; - val.data_ulong = GET_RS2S(insn, regs); - } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { + rd = riscv_insn_c_sd_extract_xs2(insn); + } else if (riscv_insn_is_c_sdsp(insn)) { len = 8; - val.data_ulong = GET_RS2C(insn, regs); -#endif - } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { + rd = riscv_insn_c_sdsp_extract_xs2(insn); + } else if (riscv_insn_is_c_sw(insn)) { len = 4; - val.data_ulong = GET_RS2S(insn, regs); - } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { + rd = riscv_insn_c_sw_extract_xs2(insn); + } else if (riscv_insn_is_c_swsp(insn)) { len = 4; - val.data_ulong = GET_RS2C(insn, regs); - } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { + rd = riscv_insn_c_swsp_extract_xs2(insn); + } else if (riscv_insn_is_c_fsd(insn)) { fp = 1; len = 8; - val.data_u64 = GET_F64_RS2S(insn, regs); - } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { + rd = riscv_insn_c_fsd_extract_fs2(insn); + } else if (riscv_insn_is_c_fsdsp(insn)) { fp = 1; len = 8; - val.data_u64 = GET_F64_RS2C(insn, regs); -#if !defined(CONFIG_64BIT) - } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { + rd = riscv_insn_c_fsdsp_extract_fs2(insn); + } else if (riscv_insn_is_c_fsw(insn)) { fp = 1; len = 4; - val.data_ulong = GET_F32_RS2S(insn, regs); - } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { + rd = riscv_insn_c_fsw_extract_fs2(insn); + } else if (riscv_insn_is_c_fswsp(insn)) { fp = 1; len = 4; - val.data_ulong = GET_F32_RS2C(insn, regs); -#endif - } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) { + rd = riscv_insn_c_fswsp_extract_fs2(insn); + } else if (riscv_insn_is_c_sh(insn)) { len = 2; - val.data_ulong = GET_RS2S(insn, regs); + rd = riscv_insn_c_sh_extract_xs2(insn); } else { - regs->epc = epc; return -1; } if (!IS_ENABLED(CONFIG_FPU) && fp) return -EOPNOTSUPP; + if (!fp) + val.data_ulong = *(unsigned long *)((unsigned long *)regs + rd); + else if (len == 8) + val.data_u64 = get_f64_rs(rd, regs); + else + val.data_ulong = get_f32_rs(rd, regs); + if (user_mode(regs)) { if (copy_to_user((u8 __user *)addr, &val, len)) return -1; diff --git a/arch/riscv/kvm/Kconfig.debug b/arch/riscv/kvm/Kconfig.debug new file mode 100644 index 00000000000000..dc76e02120a32c --- /dev/null +++ b/arch/riscv/kvm/Kconfig.debug @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "arch/riscv/kvm testing" + +config RISCV_KVM_TEST_CSR + bool "Test KVM CSR emulation" + depends on KVM + default n + help + Enable this option to enable the emulation of a test hypervisor csr. + The KVM test csr is the vsscratch register. Once this is enabled, + reading/writing to the vsscratch register will trap into the host + supervisor and reflect the change. + + If unsure, say N. + +endmenu # "arch/riscv/kvm testing" diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile index 3b8afb038b35c2..02fd27ff33eb00 100644 --- a/arch/riscv/kvm/Makefile +++ b/arch/riscv/kvm/Makefile @@ -36,6 +36,7 @@ kvm-y += vcpu_sbi_sta.o kvm-y += vcpu_sbi_system.o kvm-$(CONFIG_RISCV_SBI_V01) += vcpu_sbi_v01.o kvm-y += vcpu_switch.o +kvm-$(CONFIG_RISCV_KVM_TEST_CSR) += vcpu_test_csr.o kvm-y += vcpu_timer.o kvm-y += vcpu_vector.o kvm-y += vm.o diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index 4d89b94128aea8..d666cd24f8c0e9 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -10,9 +10,10 @@ #include #include +#include + struct insn_func { - unsigned long mask; - unsigned long match; + bool (*cmp)(u32 insn); /* * Possible return values are as follows: * 1) Returns < 0 for error case @@ -112,6 +113,9 @@ static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num, static const struct csr_func csr_funcs[] = { KVM_RISCV_VCPU_AIA_CSR_FUNCS KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS +#ifdef CONFIG_RISCV_KVM_TEST_CSR + KVM_RISCV_VCPU_TEST_CSR_FUNCS +#endif { .base = CSR_SEED, .count = 1, .func = seed_csr_rmw }, }; @@ -126,65 +130,70 @@ static const struct csr_func csr_funcs[] = { */ int kvm_riscv_vcpu_csr_return(struct kvm_vcpu *vcpu, struct kvm_run *run) { - ulong insn; - if (vcpu->arch.csr_decode.return_handled) return 0; vcpu->arch.csr_decode.return_handled = 1; /* Update destination register for CSR reads */ - insn = vcpu->arch.csr_decode.insn; - if ((insn >> SH_RD) & MASK_RX) - SET_RD(insn, &vcpu->arch.guest_context, - run->riscv_csr.ret_value); + if (vcpu->arch.csr_decode.rd) + *((ulong *)&vcpu->arch.guest_context + + vcpu->arch.csr_decode.rd) = run->riscv_csr.ret_value; /* Move to next instruction */ - vcpu->arch.guest_context.sepc += INSN_LEN(insn); + vcpu->arch.guest_context.sepc += vcpu->arch.csr_decode.insn_len; return 0; } static int csr_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn) { + #define GET_REG(_rd) (*((unsigned long *)(&vcpu->arch.guest_context) + _rd)) + int i, rc = KVM_INSN_ILLEGAL_TRAP; - unsigned int csr_num = insn >> SH_RS2; - unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX; - ulong rs1_val = GET_RS1(insn, &vcpu->arch.guest_context); + unsigned int csr_num, rd; const struct csr_func *tcfn, *cfn = NULL; ulong val = 0, wr_mask = 0, new_val = 0; /* Decode the CSR instruction */ - switch (GET_FUNCT3(insn)) { - case GET_FUNCT3(INSN_MATCH_CSRRW): + if (riscv_insn_is_csrrw(insn)) { wr_mask = -1UL; - new_val = rs1_val; - break; - case GET_FUNCT3(INSN_MATCH_CSRRS): - wr_mask = rs1_val; + new_val = GET_REG(riscv_insn_csrrw_extract_xs1(insn)); + csr_num = riscv_insn_csrrw_extract_csr(insn); + rd = riscv_insn_csrrw_extract_xd(insn); + } else if (riscv_insn_is_csrrs(insn)) { + wr_mask = GET_REG(riscv_insn_csrrs_extract_xs1(insn)); new_val = -1UL; - break; - case GET_FUNCT3(INSN_MATCH_CSRRC): - wr_mask = rs1_val; + csr_num = riscv_insn_csrrs_extract_csr(insn); + rd = riscv_insn_csrrs_extract_xd(insn); + } else if (riscv_insn_is_csrrc(insn)) { + wr_mask = GET_REG(riscv_insn_csrrc_extract_xs1(insn)); new_val = 0; - break; - case GET_FUNCT3(INSN_MATCH_CSRRWI): + csr_num = riscv_insn_csrrc_extract_csr(insn); + rd = riscv_insn_csrrc_extract_xd(insn); + } else if (riscv_insn_is_csrrwi(insn)) { wr_mask = -1UL; - new_val = rs1_num; - break; - case GET_FUNCT3(INSN_MATCH_CSRRSI): - wr_mask = rs1_num; + new_val = riscv_insn_csrrwi_extract_imm(insn); + csr_num = riscv_insn_csrrwi_extract_csr(insn); + rd = riscv_insn_csrrwi_extract_xd(insn); + } else if (riscv_insn_is_csrrsi(insn)) { + wr_mask = riscv_insn_csrrwi_extract_imm(insn); new_val = -1UL; - break; - case GET_FUNCT3(INSN_MATCH_CSRRCI): - wr_mask = rs1_num; + csr_num = riscv_insn_csrrsi_extract_csr(insn); + rd = riscv_insn_csrrsi_extract_xd(insn); + } else if (riscv_insn_is_csrrci(insn)) { + wr_mask = riscv_insn_csrrci_extract_imm(insn); new_val = 0; - break; - default: + csr_num = riscv_insn_csrrci_extract_csr(insn); + rd = riscv_insn_csrrci_extract_xd(insn); + } else { return rc; } + #undef GET_REG + /* Save instruction decode info */ - vcpu->arch.csr_decode.insn = insn; + vcpu->arch.csr_decode.rd = rd; + vcpu->arch.csr_decode.insn_len = INSN_LEN(insn); vcpu->arch.csr_decode.return_handled = 0; /* Update CSR details in kvm_run struct */ @@ -228,43 +237,39 @@ static int csr_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn) static const struct insn_func system_opcode_funcs[] = { { - .mask = INSN_MASK_CSRRW, - .match = INSN_MATCH_CSRRW, + .cmp = riscv_insn_is_csrrw, .func = csr_insn, }, { - .mask = INSN_MASK_CSRRS, - .match = INSN_MATCH_CSRRS, + .cmp = riscv_insn_is_csrrs, .func = csr_insn, }, { - .mask = INSN_MASK_CSRRC, - .match = INSN_MATCH_CSRRC, + .cmp = riscv_insn_is_csrrc, .func = csr_insn, }, { - .mask = INSN_MASK_CSRRWI, - .match = INSN_MATCH_CSRRWI, + .cmp = riscv_insn_is_csrrwi, .func = csr_insn, }, { - .mask = INSN_MASK_CSRRSI, - .match = INSN_MATCH_CSRRSI, + .cmp = riscv_insn_is_csrrsi, .func = csr_insn, }, { - .mask = INSN_MASK_CSRRCI, - .match = INSN_MATCH_CSRRCI, + .cmp = riscv_insn_is_csrrci, .func = csr_insn, }, { - .mask = INSN_MASK_WFI, - .match = INSN_MATCH_WFI, + .cmp = riscv_insn_is_wfi, .func = wfi_insn, }, { - .mask = INSN_MASK_WRS, - .match = INSN_MATCH_WRS, + .cmp = riscv_insn_is_wrs_nto, + .func = wrs_insn, + }, + { + .cmp = riscv_insn_is_wrs_sto, .func = wrs_insn, }, }; @@ -277,7 +282,7 @@ static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, for (i = 0; i < ARRAY_SIZE(system_opcode_funcs); i++) { ifn = &system_opcode_funcs[i]; - if ((insn & ifn->mask) == ifn->match) { + if (ifn->cmp(insn)) { rc = ifn->func(vcpu, run, insn); break; } @@ -375,7 +380,7 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, unsigned long htinst) { u8 data_buf[8]; - unsigned long insn; + unsigned long insn, rd; int shift = 0, len = 0, insn_len = 0; struct kvm_cpu_trap utrap = { 0 }; struct kvm_cpu_context *ct = &vcpu->arch.guest_context; @@ -407,45 +412,47 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, } /* Decode length of MMIO and shift */ - if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { + if (riscv_insn_is_lw(insn)) { len = 4; shift = 8 * (sizeof(ulong) - len); - } else if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) { + rd = riscv_insn_lw_extract_xd(insn); + } else if (riscv_insn_is_lb(insn)) { len = 1; shift = 8 * (sizeof(ulong) - len); - } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) { + rd = riscv_insn_lb_extract_xd(insn); + } else if (riscv_insn_is_lbu(insn)) { len = 1; - shift = 8 * (sizeof(ulong) - len); -#ifdef CONFIG_64BIT - } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { + rd = riscv_insn_lbu_extract_xd(insn); + } else if (riscv_insn_is_ld(insn)) { len = 8; shift = 8 * (sizeof(ulong) - len); - } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { + rd = riscv_insn_ld_extract_xd(insn); + } else if (riscv_insn_is_lwu(insn)) { len = 4; -#endif - } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { + rd = riscv_insn_lwu_extract_xd(insn); + } else if (riscv_insn_is_lh(insn)) { len = 2; shift = 8 * (sizeof(ulong) - len); - } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { + rd = riscv_insn_lh_extract_xd(insn); + } else if (riscv_insn_is_lhu(insn)) { len = 2; -#ifdef CONFIG_64BIT - } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { + rd = riscv_insn_lhu_extract_xd(insn); + } else if (riscv_insn_is_c_ld(insn)) { len = 8; shift = 8 * (sizeof(ulong) - len); - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && - ((insn >> SH_RD) & 0x1f)) { + rd = riscv_insn_c_ld_extract_xd(insn); + } else if (riscv_insn_is_c_ldsp(insn)) { len = 8; shift = 8 * (sizeof(ulong) - len); -#endif - } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { + rd = riscv_insn_c_ldsp_extract_xd(insn); + } else if (riscv_insn_is_c_lw(insn)) { len = 4; shift = 8 * (sizeof(ulong) - len); - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && - ((insn >> SH_RD) & 0x1f)) { + rd = riscv_insn_c_lw_extract_xd(insn); + } else if (riscv_insn_is_c_lwsp(insn)) { len = 4; shift = 8 * (sizeof(ulong) - len); + rd = riscv_insn_c_lwsp_extract_xd(insn); } else { return -EOPNOTSUPP; } @@ -455,7 +462,7 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, return -EIO; /* Save instruction decode info */ - vcpu->arch.mmio_decode.insn = insn; + vcpu->arch.mmio_decode.rd = rd; vcpu->arch.mmio_decode.insn_len = insn_len; vcpu->arch.mmio_decode.shift = shift; vcpu->arch.mmio_decode.len = len; @@ -498,11 +505,7 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, unsigned long fault_addr, unsigned long htinst) { - u8 data8; - u16 data16; - u32 data32; - u64 data64; - ulong data; + ulong data, rs2; unsigned long insn; int len = 0, insn_len = 0; struct kvm_cpu_trap utrap = { 0 }; @@ -534,35 +537,30 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, insn_len = INSN_LEN(insn); } - data = GET_RS2(insn, &vcpu->arch.guest_context); - data8 = data16 = data32 = data64 = data; - - if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { + if (riscv_insn_is_sw(insn)) { len = 4; - } else if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) { + rs2 = riscv_insn_sw_extract_xs2(insn); + } else if (riscv_insn_is_sb(insn)) { len = 1; -#ifdef CONFIG_64BIT - } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { + rs2 = riscv_insn_sb_extract_xs2(insn); + } else if (riscv_insn_is_sd(insn)) { len = 8; -#endif - } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { + rs2 = riscv_insn_sd_extract_xs2(insn); + } else if (riscv_insn_is_sh(insn)) { len = 2; -#ifdef CONFIG_64BIT - } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { + rs2 = riscv_insn_sh_extract_xs2(insn); + } else if (riscv_insn_is_c_sd(insn)) { len = 8; - data64 = GET_RS2S(insn, &vcpu->arch.guest_context); - } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP && - ((insn >> SH_RD) & 0x1f)) { + rs2 = riscv_insn_c_sd_extract_xs2(insn); + } else if (riscv_insn_is_c_sdsp(insn)) { len = 8; - data64 = GET_RS2C(insn, &vcpu->arch.guest_context); -#endif - } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { + rs2 = riscv_insn_c_sdsp_extract_xs2(insn); + } else if (riscv_insn_is_c_sw(insn)) { len = 4; - data32 = GET_RS2S(insn, &vcpu->arch.guest_context); - } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP && - ((insn >> SH_RD) & 0x1f)) { + rs2 = riscv_insn_c_sw_extract_xs2(insn); + } else if (riscv_insn_is_c_swsp(insn)) { len = 4; - data32 = GET_RS2C(insn, &vcpu->arch.guest_context); + rs2 = riscv_insn_c_swsp_extract_xs2(insn); } else { return -EOPNOTSUPP; } @@ -571,26 +569,24 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, if (fault_addr & (len - 1)) return -EIO; - /* Save instruction decode info */ - vcpu->arch.mmio_decode.insn = insn; vcpu->arch.mmio_decode.insn_len = insn_len; - vcpu->arch.mmio_decode.shift = 0; - vcpu->arch.mmio_decode.len = len; vcpu->arch.mmio_decode.return_handled = 0; + data = *((ulong *)(&vcpu->arch.guest_context) + rs2); + /* Copy data to kvm_run instance */ switch (len) { case 1: - *((u8 *)run->mmio.data) = data8; + *((u8 *)run->mmio.data) = data; break; case 2: - *((u16 *)run->mmio.data) = data16; + *((u16 *)run->mmio.data) = data; break; case 4: - *((u32 *)run->mmio.data) = data32; + *((u32 *)run->mmio.data) = data; break; case 8: - *((u64 *)run->mmio.data) = data64; + *((u64 *)run->mmio.data) = data; break; default: return -EOPNOTSUPP; @@ -626,18 +622,13 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, */ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) { - u8 data8; - u16 data16; - u32 data32; - u64 data64; - ulong insn; int len, shift; + unsigned long data; if (vcpu->arch.mmio_decode.return_handled) return 0; vcpu->arch.mmio_decode.return_handled = 1; - insn = vcpu->arch.mmio_decode.insn; if (run->mmio.is_write) goto done; @@ -647,29 +638,23 @@ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) switch (len) { case 1: - data8 = *((u8 *)run->mmio.data); - SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data8 << shift >> shift); + data = *((u8 *)run->mmio.data); break; case 2: - data16 = *((u16 *)run->mmio.data); - SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data16 << shift >> shift); + data = *((u16 *)run->mmio.data); break; case 4: - data32 = *((u32 *)run->mmio.data); - SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data32 << shift >> shift); + data = *((u32 *)run->mmio.data); break; case 8: - data64 = *((u64 *)run->mmio.data); - SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data64 << shift >> shift); + data = *((u64 *)run->mmio.data); break; default: return -EOPNOTSUPP; } + *((ulong *)(&vcpu->arch.guest_context) + vcpu->arch.mmio_decode.rd) = + (long)data << shift >> shift; done: /* Move to next instruction */ vcpu->arch.guest_context.sepc += vcpu->arch.mmio_decode.insn_len; diff --git a/arch/riscv/kvm/vcpu_test_csr.c b/arch/riscv/kvm/vcpu_test_csr.c new file mode 100644 index 00000000000000..b8aa503cdaba76 --- /dev/null +++ b/arch/riscv/kvm/vcpu_test_csr.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#define vcpu_to_test_csr(vcpu) (&(vcpu)->arch.test_csr) + +int kvm_riscv_vcpu_test_csr(struct kvm_vcpu *vcpu, unsigned int csr_num, + unsigned long *val, unsigned long new_val, + unsigned long wr_mask) +{ + struct kvm_test_csr *test_csr = vcpu_to_test_csr(vcpu); + + *val = test_csr->val; + + if (wr_mask) + test_csr->val = (test_csr->val & ~wr_mask) | (new_val & wr_mask); + + return KVM_INSN_CONTINUE_NEXT_SEPC; +} diff --git a/arch/riscv/tools/Makefile b/arch/riscv/tools/Makefile new file mode 100644 index 00000000000000..5f40439c12e9c1 --- /dev/null +++ b/arch/riscv/tools/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0 + +gen := arch/$(ARCH)/include/generated/asm +insn_tbl := $(src)/insn_tbl.sh +insn := $(src)/insn.tbl + +gen-y := $(gen)/insn_gen.h + +targets += $(addprefix ../../../,$(gen-y)) + +PHONY += insn + +insn: $(gen-y) + +# Create output directory if not already present +$(shell mkdir -p $(gen)) + +quiet_cmd_insn_tbl = INST_TBL $@ + cmd_insn_tbl = $(CONFIG_SHELL) $(insn_tbl) $< $@ + +$(gen)/insn_gen.h: $(insn) $(insn_tbl) FORCE + $(call if_changed,insn_tbl) diff --git a/arch/riscv/tools/insn.tbl b/arch/riscv/tools/insn.tbl new file mode 100644 index 00000000000000..56797903b141da --- /dev/null +++ b/arch/riscv/tools/insn.tbl @@ -0,0 +1,1391 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# GENERATED WITH https://github.com/riscv-software-src/riscv-unified-db +# "bundle exec rake gen:insn_table" +# +# Each line of the instruction table should have the following format: +# NAME BASE FIXED_BITS [VARIABLE_LIST] +# NAME instruction name +# BASE instruction base size (common[,(32|64)]) +# FIXED_BITS bitfields of the fixed bits of an instruction concatenated with | +# each continuous grouping of fixed bits is in the format of bits[--][<<][!...] +# if the variable requires sign extension, surround the index to sign extend at in '-' +# if the variable requires left shifting, surround the left shift amount in '<' +# a "constraint" is an integer value that is invalid for this variable +# VARIABLE_LIST a variable sized list of all variables in the instruction definition +# in the form of name[~][_MASK useful to help check if arbitrary binary is +# - riscv_insn__MATCH useful to help check if arbitrary binary is +# - riscv_insn_ useful to construct +# - riscv_insn__ useful to extract from +# +# Each line of the instruction table should have the following format: +# +# NAME BASE FIXED_BITS [VARIABLE_LIST] +# +# NAME instruction name +# BASE instruction base size (common[,(32|64)]) +# FIXED_BITS bitfields of the fixed bits of an instruction concatenated with | +# each continuous grouping of fixed bits is in the format of bits[--][<<][!...] +# if the variable requires sign extension, surround the index to sign extend at in '-' +# if the variable requires left shifting, surround the left shift amount in '<' +# a "constraint" is an integer value that is invalid for this variable +# VARIABLE_LIST a variable sized list of all variables in the instruction definition +# in the form of name[~][&2 "usage: $0 BASE INFILE OUTFILE" >&2 + echo >&2 + echo >&2 " INFILE input instruction table" + echo >&2 " OUTFILE output header file" + exit 1 +} + +if [ $# -ne 2 ]; then + usage +fi + +infile="$1" +outfile="$2" + +file=$(readlink -f $0) + +echo "/* Auto-generated rv${base} header from script arch/${file#*arch/} */" > $outfile + +echo "#ifndef RISCV_INSN_GEN_H" >> $outfile +echo "#define RISCV_INSN_GEN_H" >> $outfile +echo >> $outfile + +printf "#include " >> $outfile +echo >> $outfile +echo "#define COMMA ," >> $outfile +echo "#define SEMICOLON ;" >> $outfile +echo "#define SINGLE_ARG(...) __VA_ARGS__" >> $outfile +echo >> $outfile + +grep -E "^[a-z\.0-9]+[[:space:]]+" "$infile" | { + while read name base fixed variables; do + echo "/* $name */" + + compressed_name=${name##c.*} + invalid_inst_functions="" + variable_params="" + constraints="" + match="" + mask="" + make="" + + # All compressed instructions start with "c." + size=${compressed_name:+32}; + size=${size:-16}; + + # Replace all . with _ + formatted_inst_name=$name + while [ ! ${formatted_inst_name##*.*} ]; do + prefix=${formatted_inst_name%.*} + suffix=${formatted_inst_name##*.} + contains_dot=${formatted_inst_name##*.*} + formatted_inst_name=${contains_dot:-${prefix}_${suffix}} + done + + # Collect all fixed bits of an instruction + OLD_IFS=$IFS + IFS='|' + for segment in $fixed; do + bits=${segment%<*} + offset=${segment#*<} + + len=${#bits} + + mask="${mask} | 0b" + + while [ $len -gt 0 ]; do + len=$((len - 1)) + mask=${mask}1 + done + + if [ ${offset} -gt 0 ]; then + s=" << ${offset}" + else + s="" + fi + + mask="${mask}${s}" + + match="${match} | 0b${bits}${s}" + done + IFS=$OLD_IFS + + # Instruction only appears in one base + only_base= + if [ "${base}" != "${base%32}" ]; then + echo "#if __riscv_xlen == 32" + only_base=32 + elif [ "${base}" != "${base%64}" ]; then + echo "#if __riscv_xlen == 64" + only_base=64 + fi + + # Standard name for the instruction parameter in generated functions + insn="_insn" + + for variable in ${variables}; do + variable_name="${variable%%[<~=!]*}" + parts="${variable#*=}" + insert_mask="" + sign_extend="" + left_shift="" + extract="" + insert="" + + # Standard name for the variable parameter in generated functions + var="_${variable_name}" + variable_params="${variable_params}u32 ${var}, " + + if [ "${variable}" != "${variable#*~}" ]; then + sign_extend="1" + fi + + if [ "${variable}" != "${variable#*<}" ]; then + left_shift="${variable#*<}" + left_shift="${left_shift%%[=<~!]*}" + else + left_shift="0" + fi + + if [ "${variable}" != "${variable#*!}" ]; then + raw_constraints="${variable#*!}" + raw_constraints="${raw_constraints%%[=<~!]**}" + + OLD_IFS=$IFS + IFS='!' + for constraint in $raw_constraints; do + constraints="${constraints}(riscv_insn_${formatted_inst_name}_extract_${variable_name}(${insn}) != ${constraint}) && " + done + IFS=$OLD_IFS + fi + + offset=0 + while true; do + part=${parts##*|} + + if [ "${part#*-}" = "${part}" ]; then + high="${part}" + low="${part}" + len=1 + else + high="${part%-*}" + low="${part#*-}" + len=$((high - low + 1)) + fi + + # Don't emit shift if 0 + first_shift=${low} + if [ "${first_shift}" = "0" ]; then + first_shift= + fi + + second_shift=$((offset + left_shift)) + if [ "${second_shift}" = "0" ]; then + second_shift= + fi + + extract="${extract} | ((${insn}${first_shift:+ >> }${first_shift} & GENMASK($((len - 1)), 0))${second_shift:+ << }${second_shift})" + insert_mask="${insert_mask} & ~GENMASK(${high}, ${low})" + insert="${insert} | (((${var}${second_shift:+ >> }${second_shift}) & GENMASK($((len - 1)), 0))${first_shift:+ << }${first_shift})" + offset=$((offset + len)) + + if [ "${parts}" = "${part}" ]; then + # Processed all parts of variable + break + fi + + parts=${parts%|*} + done + + extract="${extract# | }" + + if [ ${sign_extend} ]; then + extract="sign_extend32(${extract}, ${offset})" + type="s" + else + type="u" + fi + + echo "static __always_inline ${type}${size} riscv_insn_${formatted_inst_name}_extract_${variable_name}(u${size} ${insn})" + echo "{" + echo "\treturn ${extract};" + echo "}" + echo "static __always_inline void riscv_insn_${formatted_inst_name}_insert_${variable_name}(u${size} *${insn}, ${type}32 ${var})" + echo "{" + echo "\t*_insn &= ${insert_mask# & };" + echo "\t*_insn |= ${insert# | };" + echo "}" + + if [ "${only_base}" ]; then + invalid_inst_functions="${invalid_inst_functions}static __always_inline ${type}${size} riscv_insn_${formatted_inst_name}_extract_${variable_name}(u${size} ${insn}) {\n\tpanic(\"${name} is not supported on non ${only_base}-bit systems.\");\n}\n" + fi + + make="${make} riscv_insn_${formatted_inst_name}_insert_${variable_name}(&${insn}, ${var});\n" + done + + variable_params="${variable_params%, }" + variable_params="${variable_params:-void}" + + echo "#define riscv_insn_${formatted_inst_name}_MASK (${mask# | })" + echo "#define riscv_insn_${formatted_inst_name}_MATCH (${match# | })" + echo "static __always_inline u${size} riscv_insn_${formatted_inst_name}(${variable_params})" + echo "{" + echo "\tu${size} ${insn} = riscv_insn_${formatted_inst_name}_MATCH;" + echo "${make} return ${insn};" + echo "}" + + # Check against instructions that have a variable that may contain invalid values + if [ "$constraints" ]; then + echo "__RISCV_INSN_FUNCS_CONSTRAINED(${formatted_inst_name}, ${constraints% && });" + else + echo "__RISCV_INSN_FUNCS(${formatted_inst_name});" + fi + + # If common does not appear in the base, then this instruction only appears in one base + if [ "$base" = "${base#common}" ]; then + echo "#else" + echo "__RISCV_INSN_FUNCS_UNSUPPORTED(${formatted_inst_name});" + echo "${invalid_inst_functions%\\n}" + fi + + # Instruction has a base variant + if [ "$base" != "${base%[24]}" ]; then + echo "#endif" + fi + + echo + done + + echo "#endif /* RISCV_INST_GEN_H */" +} >> $outfile diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 80364d4dbebb0c..0fc7d080f17a10 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1224,6 +1224,8 @@ enum kvm_device_type { #define KVM_DEV_TYPE_LOONGARCH_EIOINTC KVM_DEV_TYPE_LOONGARCH_EIOINTC KVM_DEV_TYPE_LOONGARCH_PCHPIC, #define KVM_DEV_TYPE_LOONGARCH_PCHPIC KVM_DEV_TYPE_LOONGARCH_PCHPIC + KVM_DEV_TYPE_TEST, +#define KVM_DEV_TYPE_TEST KVM_DEV_TYPE_TEST KVM_DEV_TYPE_MAX, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 93f356d2b3d955..612b2064167fbb 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -754,6 +754,12 @@ source "net/Kconfig.debug" endmenu # "Networking Debugging" +menu "KVM Debugging" + +source "virt/kvm/Kconfig.debug" + +endmenu # "KVM Debugging" + menu "Memory Debugging" source "mm/Kconfig.debug" diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index dc68371f76a332..f2a0dad6c14798 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -208,8 +208,10 @@ TEST_GEN_PROGS_s390 += s390/keyop TEST_GEN_PROGS_s390 += rseq_test TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) +TEST_GEN_PROGS_riscv += riscv/csr_test TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test TEST_GEN_PROGS_riscv += riscv/ebreak_test +TEST_GEN_PROGS_riscv += riscv/mmio_test TEST_GEN_PROGS_riscv += access_tracking_perf_test TEST_GEN_PROGS_riscv += arch_timer TEST_GEN_PROGS_riscv += coalesced_io_test diff --git a/tools/testing/selftests/kvm/riscv/csr_test.c b/tools/testing/selftests/kvm/riscv/csr_test.c new file mode 100644 index 00000000000000..432d043fa1ac5d --- /dev/null +++ b/tools/testing/selftests/kvm/riscv/csr_test.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * csr_test.c - Tests the csr functionality. + */ +#include "kvm_util.h" +#include "ucall_common.h" + +#define CSR_TEST 0x240 +#define FP 0x00006000 + +/* + * Use the fcsr as a U-mode accesible csr and compare against the custom 'test' + * hypervisor csr (currently using vsscratch) + */ +#define test_csr(write, initial, mode) \ +static void test_##write(void) \ +{ \ + unsigned long hypervisor_result, reference_result, old_hypervisor; \ + unsigned long mask = 0x15; \ + asm volatile ( \ + "csrs sstatus, %[enable_fp]\n" \ + "csrw fcsr, %[init]\n" \ + #write" zero, fcsr, %[mask]\n" \ + "csrr %[ref_res], fcsr\n" \ + : [ref_res] "=&r" (reference_result) \ + : [enable_fp] "r" (FP), [mask] #mode(mask), [init] "r" (initial) \ + : "memory" \ + ); \ + asm volatile ( \ + "csrw %[test_csr], %[init]\n" \ + #write" %[old], %[test_csr], %[mask]\n" \ + "csrr %[hyp_res], %[test_csr]\n" \ + : [hyp_res] "=&r" (hypervisor_result), [old] "=&r" (old_hypervisor) \ + : [test_csr] "i"(CSR_TEST), [mask] #mode(mask), [init] "r" (initial) \ + : "memory" \ + ); \ + /* Check that writing works */ \ + GUEST_ASSERT_EQ(reference_result, hypervisor_result); \ + /* Check that reading works */ \ + GUEST_ASSERT_EQ(old_hypervisor, initial); \ + GUEST_DONE(); \ +} + +test_csr(csrrw, 0x0, r) +test_csr(csrrs, 0x0, r) +test_csr(csrrc, 0x15, r) +test_csr(csrrwi, 0x0, i) +test_csr(csrrsi, 0x0, i) +test_csr(csrrci, 0x15, i) + +static void run(void *guest_code, char *instruction) +{ + struct ucall uc; + struct kvm_vm *vm; + struct kvm_vcpu *vcpu; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); + + kvm_create_device(vm, KVM_DEV_TYPE_TEST); + + vcpu_run(vcpu); + + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, + "CSR instruction '%s' failed: '%s'", instruction, + uc.buffer); + + kvm_vm_free(vm); +} + +static void check_test_csr_guest(void) +{ + unsigned long scause, stvec; + + asm volatile( + "la %[stvec], 1f\n" + "csrw stvec, %[stvec]\n" + "csrwi %[test_csr], 0x0\n" + "1:\n" + "csrr %[scause], scause\n" + : [scause] "=&r" (scause), [stvec] "=&r" (stvec) + : [test_csr] "i" (CSR_TEST) + ); + + /* An illegal instruction will be generated if CONFIG_RISCV_KVM_TEST_CSR is not enabled. */ + if (scause == 2) + GUEST_FAIL("CONFIG_RISCV_KVM_TEST_CSR not enabled.\n"); + GUEST_DONE(); +} + +static int check_test_csr(void) +{ + struct ucall uc; + struct kvm_vm *vm; + struct kvm_vcpu *vcpu; + int success; + + vm = vm_create_with_one_vcpu(&vcpu, check_test_csr_guest); + kvm_create_device(vm, KVM_DEV_TYPE_TEST); + + vcpu_run(vcpu); + + success = get_ucall(vcpu, &uc) == UCALL_DONE; + + kvm_vm_free(vm); + + return success; +} + +int main(void) +{ + /* Skip if CONFIG_RISCV_KVM_TEST_CSR not enabled */ + if (!check_test_csr()) + exit(KSFT_SKIP); + + run(test_csrrw, "csrrw"); + run(test_csrrs, "csrrs"); + run(test_csrrc, "csrrc"); + run(test_csrrwi, "csrrwi"); + run(test_csrrsi, "csrrsi"); + run(test_csrrci, "csrrci"); + + return 0; +} diff --git a/tools/testing/selftests/kvm/riscv/mmio_test.c b/tools/testing/selftests/kvm/riscv/mmio_test.c new file mode 100644 index 00000000000000..9726860a269a3f --- /dev/null +++ b/tools/testing/selftests/kvm/riscv/mmio_test.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * mmio_test.c - Tests the mmio functionality. + */ +#include "kvm_util.h" +#include "ucall_common.h" + +#define MMIO_TEST_REGION 0x20000000 + +#define test_standard_read(len, instruction, name, options) \ +static void test_##name(void) \ +{ \ + unsigned long mmio_result, reference_result; \ + /* Configure the MMIO to return 0xff for each byte to check sign extension */ \ + unsigned long reference = ((unsigned long)-1 >> ((sizeof(long) - len) * 8)); \ + *((unsigned long *)MMIO_TEST_REGION) = reference; \ + /* Check that reads through MMIO are equivalent to standard reads. */ \ + asm volatile ( \ + ".option push\n" \ + options \ + #instruction " %[mmio_res], 0(%[region])\n" \ + #instruction " %[ref_res], 0(%[ref])\n" \ + ".option pop\n" \ + : [mmio_res] "=&cr" (mmio_result), [ref_res] "=&cr" (reference_result) \ + : [region] "cr" (MMIO_TEST_REGION), [ref] "cr" (&reference) \ + ); \ + GUEST_ASSERT_EQ(mmio_result, reference_result); \ + GUEST_DONE(); \ +} + +#define test_sp_read(len, instruction, name) \ +static void test_##name(void) \ +{ \ + unsigned long mmio_result, reference_result; \ + unsigned long tmp; \ + /* Configure the MMIO to return 0xff for each byte to check sign extension */ \ + unsigned long reference = ((unsigned long)-1 >> ((sizeof(long) - len) * 8)); \ + *(((unsigned long *)MMIO_TEST_REGION)) = reference; \ + /* Check that reads through MMIO are equivalent to standard reads. */ \ + asm volatile ( \ + "mv %[tmp], sp\n" \ + "mv sp, %[region]\n" \ + ".option push\n" \ + ".option arch,+c\n" \ + #instruction " %[mmio_res], 0(sp)\n" \ + "mv sp, %[ref]\n" \ + #instruction " %[ref_res], 0(sp)\n" \ + ".option pop\n" \ + "mv sp, %[tmp]\n" \ + : [mmio_res] "=&cr" (mmio_result), [ref_res] "=&cr" (reference_result), \ + [tmp] "=&r" (tmp) \ + : [region] "r" (MMIO_TEST_REGION), [ref] "cr" (&reference) \ + ); \ + GUEST_ASSERT_EQ(mmio_result, reference_result); \ + GUEST_DONE(); \ +} + +test_standard_read(1, lb, lb, "") +test_standard_read(1, lbu, lbu, "") +test_standard_read(4, lw, lw, "") +test_standard_read(4, c.lw, c_lw, ".option arch,+c\n") +test_sp_read(4, c.lwsp, c_lwsp) + +#if __riscv_xlen == 64 +test_standard_read(2, lh, lh, "") +test_standard_read(2, lhu, lhu, "") +test_standard_read(4, lwu, lwu, "") +test_standard_read(8, ld, ld, "") +test_standard_read(8, c.ld, c_ld, ".option arch,+c\n") +test_sp_read(8, c.ldsp, c_ldsp) +#endif + +#define test_standard_write(len, write, read, name, options) \ +static void test_##name(void) \ +{ \ + unsigned long result; \ + unsigned long reference = (0x55555555UL >> ((sizeof(long) - len) * 8)); \ + /* Check that we can write and then read the same value. */ \ + asm volatile ( \ + ".option push\n" \ + options \ + #write " %[ref], 0(%[region])\n" \ + #read " %[res], 0(%[region])\n" \ + ".option pop\n" \ + : [res] "=&cr" (result) \ + : [region] "cr" (MMIO_TEST_REGION), [ref] "cr" (reference) \ + ); \ + GUEST_ASSERT_EQ(result, reference); \ + GUEST_DONE(); \ +} + +#define test_sp_write(len, write, read, name) \ +static void test_##name(void) \ +{ \ + unsigned long result; \ + unsigned long tmp; \ + unsigned long reference = (0x55555555UL >> ((sizeof(long) - len) * 8)); \ + /* Check that we can write and then read the same value. */ \ + asm volatile ( \ + "mv %[tmp], sp\n" \ + "mv sp, %[region]\n" \ + ".option push\n" \ + ".option arch,+c\n" \ + #write " %[ref], 0(sp)\n" \ + #read " %[res], 0(sp)\n" \ + ".option pop\n" \ + "mv sp, %[tmp]\n" \ + : [res] "=&cr" (result), [tmp] "=&r" (tmp) \ + : [region] "cr" (MMIO_TEST_REGION), [ref] "cr" (reference) \ + ); \ + GUEST_ASSERT_EQ(result, reference); \ + GUEST_DONE(); \ +} + +test_standard_write(1, sb, lb, sb, "") +test_standard_write(2, sh, lh, sh, "") +test_standard_write(4, sw, lw, sw, "") +test_standard_write(4, c.sw, c.lw, c_sw, ".option arch,+c\n") +test_sp_write(4, c.swsp, c.lwsp, c_swsp) + +#if __riscv_xlen == 64 +test_standard_write(8, sd, ld, sd, "") +test_standard_write(8, c.sd, c.ld, c_sd, ".option arch,+c\n") +#endif + +static void run(void *guest_code, char *instruction) +{ + struct ucall uc; + struct kvm_vm *vm; + struct kvm_vcpu *vcpu; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); + kvm_create_device(vm, KVM_DEV_TYPE_TEST); + + virt_map(vm, (unsigned long)MMIO_TEST_REGION, MMIO_TEST_REGION, 1); + + vcpu_run(vcpu); + + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, + "MMIO with instruction '%s' failed: '%s'", instruction, + uc.buffer); + + kvm_vm_free(vm); +} + +void test_mmio_read_sign_extension(void) +{ + run(test_lb, "lb"); + run(test_lbu, "lbu"); + run(test_lw, "lw"); + run(test_c_lw, "c.lw"); + run(test_c_lwsp, "c.lwsp"); + +#if __riscv_xlen == 64 + run(test_lh, "lh"); + run(test_lhu, "lhu"); + run(test_lwu, "lwu"); + run(test_ld, "ld"); + run(test_c_ld, "c.ld"); + run(test_c_ldsp, "c.ldsp"); +#endif +} + +void test_mmio_write(void) +{ + run(test_sb, "sb"); + run(test_sh, "sh"); + run(test_sw, "sw"); + run(test_c_sw, "c.sw"); + run(test_c_swsp, "c.swsp"); + +#if __riscv_xlen == 64 + run(test_sd, "sd"); + run(test_c_sd, "c.sd"); +#endif +} + +int main(void) +{ + test_mmio_read_sign_extension(); + test_mmio_write(); + + return 0; +} diff --git a/virt/kvm/Kconfig.debug b/virt/kvm/Kconfig.debug new file mode 100644 index 00000000000000..d24709f5bcbffc --- /dev/null +++ b/virt/kvm/Kconfig.debug @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config KVM_MMIO_TEST + bool "Enable kvm mmio testing" + depends on KVM + depends on KVM_MMIO + default n + help + Enable testing for kvm mmio. This is a test-only mmio device that + stores writes in a buffer and returns the buffered data on a read. + + This is useful for testing the kvm mmio emulation code. Enabling + this does not run any tests, just builds in the support for the test + device into the kernel. + + If unsure, say N. diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm index d047d4cf58c9fd..bd4da8c2392346 100644 --- a/virt/kvm/Makefile.kvm +++ b/virt/kvm/Makefile.kvm @@ -8,6 +8,7 @@ KVM ?= ../../../virt/kvm kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o kvm-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o kvm-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o +kvm-$(CONFIG_KVM_MMIO_TEST) += $(KVM)/mmio_test.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 9093251beb3980..a58a894b0353d7 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -59,6 +59,7 @@ #include "async_pf.h" #include "kvm_mm.h" #include "vfio.h" +#include "mmio_test.h" #include @@ -6532,6 +6533,10 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) if (WARN_ON_ONCE(r)) goto err_vfio; + r = kvm_mmio_test_ops_init(); + if (WARN_ON_ONCE(r)) + goto err_mmio_test; + r = kvm_gmem_init(module); if (r) goto err_gmem; @@ -6559,6 +6564,8 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) err_gmem: kvm_vfio_ops_exit(); err_vfio: + kvm_mmio_test_ops_exit(); +err_mmio_test: kvm_async_pf_deinit(); err_async_pf: kvm_irqfd_exit(); @@ -6589,6 +6596,7 @@ void kvm_exit(void) free_cpumask_var(per_cpu(cpu_kick_mask, cpu)); kmem_cache_destroy(kvm_vcpu_cache); kvm_gmem_exit(); + kvm_mmio_test_ops_exit(); kvm_vfio_ops_exit(); kvm_async_pf_deinit(); kvm_irqfd_exit(); diff --git a/virt/kvm/mmio_test.c b/virt/kvm/mmio_test.c new file mode 100644 index 00000000000000..fa84c2b4c5fcfe --- /dev/null +++ b/virt/kvm/mmio_test.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * mmio_test.c - Kernel module side for testing the KVM riscv mmio functionality. + */ + +#include + +#include +#include "mmio_test.h" + +struct mmio_test { + struct kvm *kvm; + struct kvm_io_device dev; + unsigned long start; + unsigned long size; + char cache[16]; +}; + +static struct mmio_test *kvm_to_mmio_test_dev(const struct kvm_io_device *dev) +{ + return container_of(dev, struct mmio_test, dev); +} + +static int mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + struct mmio_test *mmio_test = kvm_to_mmio_test_dev(dev); + + if ((addr - mmio_test->start) >= mmio_test->size) + return -1; + + /* Write back cached value */ + memcpy(val, &mmio_test->cache[(addr - mmio_test->start)], len); + return 0; +} + +static int mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + struct mmio_test *mmio_test = kvm_to_mmio_test_dev(dev); + + if ((addr - mmio_test->start) >= mmio_test->size) + return -1; + + /* Cache value */ + memcpy(&mmio_test->cache[(addr - mmio_test->start)], val, len); + return 0; +} + +static const struct kvm_io_device_ops mmio_ops = { + .read = mmio_read, + .write = mmio_write, +}; + +static int mmio_test_create(struct kvm_device *dev, u32 type) +{ + struct mmio_test *mmio_test; + + mmio_test = kzalloc(sizeof(*mmio_test), GFP_KERNEL); + if (!mmio_test) + return -ENOMEM; + + mmio_test->start = 0x20000000; + mmio_test->size = 0x16; + + dev->private = mmio_test; + + kvm_iodevice_init(&mmio_test->dev, &mmio_ops); + kvm_io_bus_register_dev(dev->kvm, KVM_MMIO_BUS, mmio_test->start, + mmio_test->size, &mmio_test->dev); + + return 0; +} + +static void mmio_test_release(struct kvm_device *dev) +{ + kfree(dev->private); +} + +struct kvm_device_ops kvm_riscv_mmio_test_device_ops = { + .name = "kvm-riscv-mmio_test", + .create = mmio_test_create, + .release = mmio_test_release, +}; + +int kvm_mmio_test_ops_init(void) +{ + return kvm_register_device_ops(&kvm_riscv_mmio_test_device_ops, + KVM_DEV_TYPE_TEST); +} + +void kvm_mmio_test_ops_exit(void) +{ + kvm_unregister_device_ops(KVM_DEV_TYPE_TEST); +} diff --git a/virt/kvm/mmio_test.h b/virt/kvm/mmio_test.h new file mode 100644 index 00000000000000..49a6e900eec9ab --- /dev/null +++ b/virt/kvm/mmio_test.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __KVM_MMIO_TEST_H +#define __KVM_MMIO_TEST_H + +#ifdef CONFIG_KVM_MMIO_TEST +int kvm_mmio_test_ops_init(void); +void kvm_mmio_test_ops_exit(void); +#else +static inline int kvm_mmio_test_ops_init(void) +{ + return 0; +} +static inline void kvm_mmio_test_ops_exit(void) +{ +} +#endif + +#endif