diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h index 1f37b600ca4792..9e39b0e1516908 100644 --- a/arch/riscv/include/asm/kvm_aia.h +++ b/arch/riscv/include/asm/kvm_aia.h @@ -112,6 +112,7 @@ int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type); int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level); int kvm_riscv_aia_aplic_init(struct kvm *kvm); void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm); +bool kvm_riscv_aia_imsic_state_hw_backed(struct kvm_vcpu *vcpu); #ifdef CONFIG_32BIT void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu); @@ -141,12 +142,23 @@ int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, unsigned long *val, unsigned long new_val, unsigned long wr_mask); +int kvm_riscv_vcpu_aia_hstateen_enable(struct kvm_vcpu *vcpu, + unsigned int csr_num, unsigned long *val, + unsigned long new_val, unsigned long wr_mask); +int kvm_riscv_vcpu_aia_rmw_isel(struct kvm_vcpu *vcpu, unsigned int csr_num, unsigned long *val, + unsigned long new_val, unsigned long wr_mask); int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, unsigned long *val, unsigned long new_val, unsigned long wr_mask); #define KVM_RISCV_VCPU_AIA_CSR_FUNCS \ { .base = CSR_SIREG, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_ireg }, \ -{ .base = CSR_STOPEI, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_topei }, +{ .base = CSR_SISELECT, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_isel }, \ +{ .base = CSR_STOPEI, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_topei }, \ +{ .base = CSR_STOPI, .count = 1, .func = kvm_riscv_vcpu_aia_hstateen_enable }, \ + +#define KVM_RISCV_VCPU_AIA_CSR_32BIT_FUNCS \ +{ .base = CSR_SIPH, .count = 1, .func = kvm_riscv_vcpu_aia_hstateen_enable }, \ +{ .base = CSR_SIEH, .count = 1, .func = kvm_riscv_vcpu_aia_hstateen_enable }, \ int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu); diff --git a/arch/riscv/include/asm/kvm_vcpu_insn.h b/arch/riscv/include/asm/kvm_vcpu_insn.h index 350011c83581cb..1125f3f1c8c434 100644 --- a/arch/riscv/include/asm/kvm_vcpu_insn.h +++ b/arch/riscv/include/asm/kvm_vcpu_insn.h @@ -6,6 +6,8 @@ #ifndef __KVM_VCPU_RISCV_INSN_H #define __KVM_VCPU_RISCV_INSN_H +#include + struct kvm_vcpu; struct kvm_run; struct kvm_cpu_trap; @@ -44,5 +46,7 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, unsigned long fault_addr, unsigned long htinst); int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); +int kvm_riscv_vcpu_hstateen_lazy_enable(struct kvm_vcpu *vcpu, unsigned int csr_num, + uint64_t hstateen_feature_bit_mask); #endif diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index 19afd1f2353776..3dfabf51a4d2b3 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -235,12 +235,48 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, return 0; } +int kvm_riscv_vcpu_aia_hstateen_enable(struct kvm_vcpu *vcpu, + unsigned int csr_num, + unsigned long *val, + unsigned long new_val, + unsigned long wr_mask) +{ + /* If AIA not available then redirect trap */ + if (!kvm_riscv_aia_available()) + return KVM_INSN_ILLEGAL_TRAP; + + /* If AIA not initialized then forward to user space */ + if (!kvm_riscv_aia_initialized(vcpu->kvm)) + return KVM_INSN_EXIT_TO_USER_SPACE; + + return kvm_riscv_vcpu_hstateen_lazy_enable(vcpu, csr_num, SMSTATEEN0_AIA); +} + +int kvm_riscv_vcpu_aia_rmw_isel(struct kvm_vcpu *vcpu, + unsigned int csr_num, + unsigned long *val, + unsigned long new_val, + unsigned long wr_mask) +{ + /* If AIA not available then redirect trap */ + if (!kvm_riscv_aia_available()) + return KVM_INSN_ILLEGAL_TRAP; + + /* If AIA not initialized then forward to user space */ + if (!kvm_riscv_aia_initialized(vcpu->kvm)) + return KVM_INSN_EXIT_TO_USER_SPACE; + + return kvm_riscv_vcpu_hstateen_lazy_enable(vcpu, csr_num, SMSTATEEN0_AIA_ISEL); +} + int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, unsigned int csr_num, unsigned long *val, unsigned long new_val, unsigned long wr_mask) { + bool vsfile_present = kvm_riscv_aia_imsic_state_hw_backed(vcpu); + /* If AIA not available then redirect trap */ if (!kvm_riscv_aia_available()) return KVM_INSN_ILLEGAL_TRAP; @@ -249,6 +285,26 @@ int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, if (!kvm_riscv_aia_initialized(vcpu->kvm)) return KVM_INSN_EXIT_TO_USER_SPACE; + /* Continue if smstaeen is not present */ + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + goto skip_hstateen; + + /* Enable the bit in hstateen0 lazily upon first access */ + if (!(vcpu->arch.cfg.hstateen0 & SMSTATEEN0_AIA_IMSIC)) { + vcpu->arch.cfg.hstateen0 |= SMSTATEEN0_AIA_IMSIC; + if (IS_ENABLED(CONFIG_32BIT)) + csr_set(CSR_HSTATEEN0H, SMSTATEEN0_AIA_IMSIC >> 32); + else + csr_set(CSR_HSTATEEN0, SMSTATEEN0_AIA_IMSIC); + if (vsfile_present) + return KVM_INSN_CONTINUE_SAME_SEPC; + } else if (vsfile_present) { + pr_err("Unexpected trap for CSR [%x] with hstateen0 enabled and valid vsfile\n", + csr_num); + return KVM_INSN_EXIT_TO_USER_SPACE; + } + +skip_hstateen: return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, val, new_val, wr_mask); } @@ -400,11 +456,32 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, unsigned long wr_mask) { unsigned int isel; + bool vsfile_present = kvm_riscv_aia_imsic_state_hw_backed(vcpu); /* If AIA not available then redirect trap */ if (!kvm_riscv_aia_available()) return KVM_INSN_ILLEGAL_TRAP; + /* Continue if smstaeen is not present */ + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + goto skip_hstateen; + + /* Enable the bit in hstateen0 lazily upon first access */ + if (!(vcpu->arch.cfg.hstateen0 & SMSTATEEN0_AIA_ISEL)) { + vcpu->arch.cfg.hstateen0 |= SMSTATEEN0_AIA_ISEL; + if (IS_ENABLED(CONFIG_32BIT)) + csr_set(CSR_HSTATEEN0H, SMSTATEEN0_AIA_ISEL >> 32); + else + csr_set(CSR_HSTATEEN0, SMSTATEEN0_AIA_ISEL); + if (vsfile_present) + return KVM_INSN_CONTINUE_SAME_SEPC; + } else if (vsfile_present) { + pr_err("Unexpected trap for CSR [%x] with hstateen0 enabled and valid vsfile\n", + csr_num); + return KVM_INSN_EXIT_TO_USER_SPACE; + } + +skip_hstateen: /* First try to emulate in kernel space */ isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK; if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index 29ef9c2133a933..d8e6f14850c05a 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -361,6 +361,14 @@ static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix, return 0; } +bool kvm_riscv_aia_imsic_state_hw_backed(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context; + struct imsic *imsic = vaia->imsic_state; + + return imsic && imsic->vsfile_cpu >= 0; +} + struct imsic_vsfile_read_data { int hgei; u32 nr_eix; diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 60d684c76c5873..0aaa9e0e4a01ec 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -560,16 +560,6 @@ static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu) !riscv_isa_extension_available(isa, SVADE)) cfg->henvcfg |= ENVCFG_ADUE; - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; - if (riscv_isa_extension_available(isa, SSAIA)) - cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | - SMSTATEEN0_AIA | - SMSTATEEN0_AIA_ISEL; - if (riscv_isa_extension_available(isa, SMSTATEEN)) - cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; - } - cfg->hedeleg = KVM_HEDELEG_DEFAULT; if (vcpu->guest_debug) cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index 97dec18e69892a..6f2bba7533cf2e 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -235,10 +235,71 @@ static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num, return KVM_INSN_EXIT_TO_USER_SPACE; } +int kvm_riscv_vcpu_hstateen_lazy_enable(struct kvm_vcpu *vcpu, unsigned int csr_num, + uint64_t hstateen_feature_bit_mask) +{ + /* Access from VS shouldn't trap if smstaeen is not present */ + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + return KVM_INSN_EXIT_TO_USER_SPACE; + + /* + * Make sure that KVM doesn't enable any guest visible state via sstateen (lower 32 bits) + * yet. Access is restricted to prevent unintended behavior. + */ + if (hstateen_feature_bit_mask & GENMASK(31, 0)) { + pr_err("Unexpected access from lower 32 bits of hstateen0\n"); + return KVM_INSN_EXIT_TO_USER_SPACE; + } + + /* Enable the bit in hstateen0 lazily upon first access */ + if (!(vcpu->arch.cfg.hstateen0 & hstateen_feature_bit_mask)) { + vcpu->arch.cfg.hstateen0 |= hstateen_feature_bit_mask; + csr_set(CSR_HSTATEEN0, hstateen_feature_bit_mask); + if (IS_ENABLED(CONFIG_32BIT)) + csr_set(CSR_HSTATEEN0H, hstateen_feature_bit_mask >> 32); + } else { + return KVM_INSN_EXIT_TO_USER_SPACE; + } + + /* Let the guest retry the instruction read after hstateen0 is modified */ + return KVM_INSN_CONTINUE_SAME_SEPC; +} + +static int kvm_riscv_vcpu_hstateen_enable_senvcfg(struct kvm_vcpu *vcpu, + unsigned int csr_num, + unsigned long *val, + unsigned long new_val, + unsigned long wr_mask) +{ + return kvm_riscv_vcpu_hstateen_lazy_enable(vcpu, csr_num, SMSTATEEN0_HSENVCFG); +} + +static int kvm_riscv_vcpu_hstateen_enable_stateen(struct kvm_vcpu *vcpu, + unsigned int csr_num, + unsigned long *val, + unsigned long new_val, + unsigned long wr_mask) +{ + const unsigned long *isa = vcpu->arch.isa; + + if (riscv_isa_extension_available(isa, SMSTATEEN)) + return kvm_riscv_vcpu_hstateen_lazy_enable(vcpu, csr_num, SMSTATEEN0_SSTATEEN0); + else + return KVM_INSN_EXIT_TO_USER_SPACE; +} + +#define KVM_RISCV_VCPU_STATEEN_CSR_FUNCS \ +{ .base = CSR_SENVCFG, .count = 1, .func = kvm_riscv_vcpu_hstateen_enable_senvcfg }, \ +{ .base = CSR_SSTATEEN0, .count = 1, .func = kvm_riscv_vcpu_hstateen_enable_stateen },\ + static const struct csr_func csr_funcs[] = { KVM_RISCV_VCPU_AIA_CSR_FUNCS KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS + KVM_RISCV_VCPU_STATEEN_CSR_FUNCS { .base = CSR_SEED, .count = 1, .func = seed_csr_rmw }, +#ifdef CONFIG_32BIT + KVM_RISCV_VCPU_AIA_CSR_32BIT_FUNCS +#endif }; /**