Skip to content

[PW_SID:1102437] riscv: optimize Vector context restore on syscall #2028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
linux-riscv-bot wants to merge 1 commit into
base: workflow__riscv__fixes
Choose a base branch
from
Open

[PW_SID:1102437] riscv: optimize Vector context restore on syscall #2028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions arch/riscv/include/asm/kvm_vcpu_vector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@
#include <asm/vector.h>
#include <asm/kvm_host.h>

static __always_inline void __kvm_riscv_vector_save(struct kvm_cpu_context *context)
static __always_inline void kvm_riscv_vector_save(struct kvm_cpu_context *context)
{
riscv_v_enable();
__riscv_v_vstate_save(&context->vector, context->vector.datap);
riscv_v_disable();
}

static __always_inline void __kvm_riscv_vector_restore(struct kvm_cpu_context *context)
static __always_inline void kvm_riscv_vector_restore(struct kvm_cpu_context *context)
{
riscv_v_enable();
__riscv_v_vstate_restore(&context->vector, context->vector.datap);
riscv_v_disable();
}

void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu);
Expand Down
15 changes: 8 additions & 7 deletions arch/riscv/include/asm/vector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,6 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
{
unsigned long vl;

riscv_v_enable();
__vstate_csr_save(save_to);
if (has_xtheadvector()) {
asm volatile (
Expand Down Expand Up @@ -227,15 +226,13 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
".option pop\n\t"
: "=&r" (vl) : "r" (datap) : "memory");
}
riscv_v_disable();
}

static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from,
void *datap)
{
unsigned long vl;

riscv_v_enable();
if (has_xtheadvector()) {
asm volatile (
"mv t0, %0\n\t"
Expand Down Expand Up @@ -264,14 +261,12 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_
: "=&r" (vl) : "r" (datap) : "memory");
}
__vstate_csr_restore(restore_from);
riscv_v_disable();
}

static inline void __riscv_v_vstate_discard(void)
{
unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1);

riscv_v_enable();
if (has_xtheadvector())
asm volatile (THEAD_VSETVLI_T4X0E8M8D1 : : : "t4");
else
Expand All @@ -291,14 +286,14 @@ static inline void __riscv_v_vstate_discard(void)
"vsetvl %0, x0, %1\n\t"
".option pop\n\t"
: "=&r" (vl) : "r" (vtype_inval));

riscv_v_disable();
}

static inline void riscv_v_vstate_discard(struct pt_regs *regs)
{
if (riscv_v_vstate_query(regs)) {
riscv_v_enable();
__riscv_v_vstate_discard();
riscv_v_disable();
__riscv_v_vstate_dirty(regs);
}
}
Expand All @@ -307,7 +302,9 @@ static inline void riscv_v_vstate_save(struct __riscv_v_ext_state *vstate,
struct pt_regs *regs)
{
if (__riscv_v_vstate_check(regs->status, DIRTY)) {
riscv_v_enable();
__riscv_v_vstate_save(vstate, vstate->datap);
riscv_v_disable();
__riscv_v_vstate_clean(regs);
}
}
Expand All @@ -316,7 +313,9 @@ static inline void riscv_v_vstate_restore(struct __riscv_v_ext_state *vstate,
struct pt_regs *regs)
{
if (riscv_v_vstate_query(regs)) {
riscv_v_enable();
__riscv_v_vstate_restore(vstate, vstate->datap);
riscv_v_disable();
__riscv_v_vstate_clean(regs);
}
}
Expand Down Expand Up @@ -378,8 +377,10 @@ static inline void __switch_to_vector(struct task_struct *prev,
prev->thread.riscv_v_flags |= RISCV_PREEMPT_V_IN_SCHEDULE;
}
if (riscv_preempt_v_dirty(prev)) {
riscv_v_enable();
__riscv_v_vstate_save(&prev->thread.kernel_vstate,
prev->thread.kernel_vstate.datap);
riscv_v_disable();
riscv_preempt_v_clear_dirty(prev);
}
} else {
Expand Down
29 changes: 23 additions & 6 deletions arch/riscv/kernel/kernel_mode_vector.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ static int riscv_v_stop_kernel_context(void)

static int riscv_v_start_kernel_context(bool *is_nested)
{
struct __riscv_v_ext_state *kvstate, *uvstate;
struct __riscv_v_ext_state *kvstate;

kvstate = &current->thread.kernel_vstate;
if (!kvstate->datap)
Expand All @@ -134,20 +134,35 @@ static int riscv_v_start_kernel_context(bool *is_nested)
*is_nested = true;
get_cpu_vector_context();
if (riscv_preempt_v_dirty(current)) {
riscv_v_enable();
__riscv_v_vstate_save(kvstate, kvstate->datap);
riscv_v_disable();
riscv_preempt_v_clear_dirty(current);
}
riscv_preempt_v_set_restore(current);
return 0;
}

/* Transfer the ownership of V from user to kernel, then save */
riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY);
/*
* Skip saving user's context if it is not DIRTY. We would have to start KMV in "dirty" if
* this check is performed after KMV starts, to protect user's ctx. Then, we could waste
* time saving already "clean" context once KMV is started in "dirty".
*/
if (__riscv_v_vstate_check(task_pt_regs(current)->status, DIRTY)) {
uvstate = &current->thread.vstate;
__riscv_v_vstate_save(uvstate, uvstate->datap);
/* Transfer the ownership of V from user to kernel, then save */
riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY);
/*
* Calling the guarded version of vstate_save to make the code cleaner. Also, the
* vstate check within the call is necessary as context switch may happen between
* __riscv_v_vstate_check and riscv_v_start. In such case we are not supposed to
* save the context again.
*/
riscv_v_vstate_save(&current->thread.vstate, task_pt_regs(current));
riscv_preempt_v_clear_dirty(current);
return 0;
}
riscv_preempt_v_clear_dirty(current);

riscv_v_start(RISCV_PREEMPT_V);
return 0;
}

Expand Down Expand Up @@ -180,7 +195,9 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs)
depth = riscv_v_ctx_get_depth();
if (depth == 0) {
if (riscv_preempt_v_restore(current)) {
riscv_v_enable();
__riscv_v_vstate_restore(vstate, vstate->datap);
riscv_v_disable();
__riscv_v_vstate_clean(regs);
riscv_preempt_v_reset_flags();
}
Expand Down
8 changes: 4 additions & 4 deletions arch/riscv/kvm/vcpu_vector.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx,
{
if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) {
if (riscv_isa_extension_available(isa, v))
__kvm_riscv_vector_save(cntx);
kvm_riscv_vector_save(cntx);
kvm_riscv_vcpu_vector_clean(cntx);
}
}
Expand All @@ -56,7 +56,7 @@ void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx,
{
if ((cntx->sstatus & SR_VS) != SR_VS_OFF) {
if (riscv_isa_extension_available(isa, v))
__kvm_riscv_vector_restore(cntx);
kvm_riscv_vector_restore(cntx);
kvm_riscv_vcpu_vector_clean(cntx);
}
}
Expand All @@ -65,13 +65,13 @@ void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx)
{
/* No need to check host sstatus as it can be modified outside */
if (!kvm_riscv_isa_check_host(V))
__kvm_riscv_vector_save(cntx);
kvm_riscv_vector_save(cntx);
}

void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx)
{
if (!kvm_riscv_isa_check_host(V))
__kvm_riscv_vector_restore(cntx);
kvm_riscv_vector_restore(cntx);
}

int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu)
Expand Down