diff --git a/MAINTAINERS b/MAINTAINERS index 69511c3b2b76fb..7ceae95488f9a5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20833,6 +20833,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml F: drivers/iommu/riscv/ +RISC-V FIRMWARE DRIVERS +M: Conor Dooley +L: linux-riscv@lists.infradead.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git +F: drivers/firmware/riscv/* + RISC-V MICROCHIP FPGA SUPPORT M: Conor Dooley M: Daire McNamara @@ -20897,6 +20904,14 @@ F: arch/riscv/boot/dts/spacemit/ N: spacemit K: spacemit +RISC-V SSE DRIVER +M: Clément Léger +R: Himanshu Chauhan +L: linux-riscv@lists.infradead.org +S: Maintained +F: drivers/firmware/riscv/riscv_sse.c +F: include/linux/riscv_sse.h + RISC-V THEAD SoC SUPPORT M: Drew Fustini M: Guo Ren diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index a8a2af6dfe9d24..982c4be9a9c3ee 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -90,16 +90,24 @@ #define PER_CPU_OFFSET_SHIFT 3 #endif -.macro asm_per_cpu dst sym tmp - REG_L \tmp, TASK_TI_CPU_NUM(tp) - slli \tmp, \tmp, PER_CPU_OFFSET_SHIFT +.macro asm_per_cpu_with_cpu dst sym tmp cpu + slli \tmp, \cpu, PER_CPU_OFFSET_SHIFT la \dst, __per_cpu_offset add \dst, \dst, \tmp REG_L \tmp, 0(\dst) la \dst, \sym add \dst, \dst, \tmp .endm + +.macro asm_per_cpu dst sym tmp + REG_L \tmp, TASK_TI_CPU_NUM(tp) + asm_per_cpu_with_cpu \dst \sym \tmp \tmp +.endm #else /* CONFIG_SMP */ +.macro asm_per_cpu_with_cpu dst sym tmp cpu + la \dst, \sym +.endm + .macro asm_per_cpu dst sym tmp la \dst, \sym .endm diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 3d250824178bd5..4d7f81c620efee 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -35,6 +35,7 @@ enum sbi_ext_id { SBI_EXT_DBCN = 0x4442434E, SBI_EXT_STA = 0x535441, SBI_EXT_NACL = 0x4E41434C, + SBI_EXT_SSE = 0x535345, /* Experimentals extensions must lie within this range */ SBI_EXT_EXPERIMENTAL_START = 0x08000000, @@ -402,6 +403,66 @@ enum sbi_ext_nacl_feature { #define SBI_NACL_SHMEM_SRET_X(__i) ((__riscv_xlen / 8) * (__i)) #define SBI_NACL_SHMEM_SRET_X_LAST 31 +enum sbi_ext_sse_fid { + SBI_SSE_EVENT_ATTR_READ = 0, + SBI_SSE_EVENT_ATTR_WRITE, + SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_UNREGISTER, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE, + SBI_SSE_EVENT_COMPLETE, + SBI_SSE_EVENT_SIGNAL, + SBI_SSE_EVENT_HART_UNMASK, + SBI_SSE_EVENT_HART_MASK, +}; + +enum sbi_sse_state { + SBI_SSE_STATE_UNUSED = 0, + SBI_SSE_STATE_REGISTERED = 1, + SBI_SSE_STATE_ENABLED = 2, + SBI_SSE_STATE_RUNNING = 3, +}; + +/* SBI SSE Event Attributes. */ +enum sbi_sse_attr_id { + SBI_SSE_ATTR_STATUS = 0x00000000, + SBI_SSE_ATTR_PRIO = 0x00000001, + SBI_SSE_ATTR_CONFIG = 0x00000002, + SBI_SSE_ATTR_PREFERRED_HART = 0x00000003, + SBI_SSE_ATTR_ENTRY_PC = 0x00000004, + SBI_SSE_ATTR_ENTRY_ARG = 0x00000005, + SBI_SSE_ATTR_INTERRUPTED_SEPC = 0x00000006, + SBI_SSE_ATTR_INTERRUPTED_FLAGS = 0x00000007, + SBI_SSE_ATTR_INTERRUPTED_A6 = 0x00000008, + SBI_SSE_ATTR_INTERRUPTED_A7 = 0x00000009, + + SBI_SSE_ATTR_MAX = 0x0000000A +}; + +#define SBI_SSE_ATTR_STATUS_STATE_OFFSET 0 +#define SBI_SSE_ATTR_STATUS_STATE_MASK 0x3 +#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET 2 +#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET 3 + +#define SBI_SSE_ATTR_CONFIG_ONESHOT (1 << 0) + +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPP (1 << 0) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPIE (1 << 1) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV (1 << 2) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP (1 << 3) + +#define SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS 0x00000000 +#define SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP 0x00000001 +#define SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS 0x00008000 +#define SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW 0x00010000 +#define SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS 0x00100000 +#define SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS 0x00108000 +#define SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED 0xffff0000 +#define SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED 0xffff8000 + +#define SBI_SSE_EVENT_PLATFORM (1 << 14) +#define SBI_SSE_EVENT_GLOBAL (1 << 15) + /* SBI spec version fields */ #define SBI_SPEC_VERSION_DEFAULT 0x1 #define SBI_SPEC_VERSION_MAJOR_SHIFT 24 @@ -419,6 +480,8 @@ enum sbi_ext_nacl_feature { #define SBI_ERR_ALREADY_STARTED -7 #define SBI_ERR_ALREADY_STOPPED -8 #define SBI_ERR_NO_SHMEM -9 +#define SBI_ERR_INVALID_STATE -10 +#define SBI_ERR_BAD_RANGE -11 extern unsigned long sbi_spec_version; struct sbiret { @@ -505,6 +568,8 @@ static inline int sbi_err_map_linux_errno(int err) case SBI_ERR_DENIED: return -EPERM; case SBI_ERR_INVALID_PARAM: + case SBI_ERR_BAD_RANGE: + case SBI_ERR_INVALID_STATE: return -EINVAL; case SBI_ERR_INVALID_ADDRESS: return -EFAULT; diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index 0e45db78b24bf2..62344daad73dc6 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -18,6 +18,11 @@ load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp .endm +/* Load the per-CPU IRQ shadow call stack to gp. */ +.macro scs_load_sse_stack reg_evt + REG_L gp, SSE_REG_EVT_SHADOW_STACK(\reg_evt) +.endm + /* Load task_scs_sp(current) to gp. */ .macro scs_load_current REG_L gp, TASK_TI_SCS_SP(tp) @@ -41,6 +46,8 @@ .endm .macro scs_load_irq_stack tmp .endm +.macro scs_load_sse_stack reg_evt +.endm .macro scs_load_current .endm .macro scs_load_current_if_task_changed prev diff --git a/arch/riscv/include/asm/sse.h b/arch/riscv/include/asm/sse.h new file mode 100644 index 00000000000000..aaddda77f5b607 --- /dev/null +++ b/arch/riscv/include/asm/sse.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Rivos Inc. + */ +#ifndef __ASM_SSE_H +#define __ASM_SSE_H + +#include + +#ifdef CONFIG_RISCV_SSE + +struct sse_event_interrupted_state { + unsigned long a6; + unsigned long a7; +}; + +struct sse_event_arch_data { + void *stack; + void *shadow_stack; + unsigned long tmp; + struct sse_event_interrupted_state interrupted; + unsigned long interrupted_state_phys; + u32 evt_id; +}; + +static inline bool sse_event_is_global(u32 evt) +{ + return !!(evt & SBI_SSE_EVENT_GLOBAL); +} + +struct sse_registered_event; +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, + int cpu); +void arch_sse_free_event(struct sse_event_arch_data *arch_evt); +int arch_sse_register_event(struct sse_event_arch_data *arch_evt); + +void sse_handle_event(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs); +asmlinkage void handle_sse(void); +asmlinkage void do_sse(struct sse_event_arch_data *arch_evt, + struct pt_regs *reg); + +#endif + +#endif diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920ca..cd1cead0c682d3 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -88,6 +88,19 @@ static inline void __switch_to_envcfg(struct task_struct *next) :: "r" (next->thread.envcfg) : "memory"); } +#ifdef CONFIG_RISCV_SSE +DECLARE_PER_CPU(struct task_struct *, __sse_entry_task); + +static inline void __switch_sse_entry_task(struct task_struct *next) +{ + __this_cpu_write(__sse_entry_task, next); +} +#else +static inline void __switch_sse_entry_task(struct task_struct *next) +{ +} +#endif + extern struct task_struct *__switch_to(struct task_struct *, struct task_struct *); @@ -122,6 +135,7 @@ do { \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ + __switch_sse_entry_task(__next); \ ((last) = __switch_to(__prev, __next)); \ } while (0) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index f5916a70879a87..28e9805e61fc01 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -36,6 +36,7 @@ #define OVERFLOW_STACK_SIZE SZ_4K #define IRQ_STACK_SIZE THREAD_SIZE +#define SSE_STACK_SIZE THREAD_SIZE #ifndef __ASSEMBLY__ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index f7480c9c6f8d73..d347768d690d84 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -99,6 +99,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o obj-$(CONFIG_RISCV_SBI) += sbi.o sbi_ecall.o +obj-$(CONFIG_RISCV_SSE) += sse.o sse_entry.o ifeq ($(CONFIG_RISCV_SBI), y) obj-$(CONFIG_SMP) += sbi-ipi.o obj-$(CONFIG_SMP) += cpu_ops_sbi.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 16490755304e0c..7b2d0480f772e4 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include void asm_offsets(void); @@ -510,4 +512,14 @@ void asm_offsets(void) DEFINE(FREGS_A6, offsetof(struct __arch_ftrace_regs, a6)); DEFINE(FREGS_A7, offsetof(struct __arch_ftrace_regs, a7)); #endif + +#ifdef CONFIG_RISCV_SSE + OFFSET(SSE_REG_EVT_STACK, sse_event_arch_data, stack); + OFFSET(SSE_REG_EVT_SHADOW_STACK, sse_event_arch_data, shadow_stack); + OFFSET(SSE_REG_EVT_TMP, sse_event_arch_data, tmp); + + DEFINE(SBI_EXT_SSE, SBI_EXT_SSE); + DEFINE(SBI_SSE_EVENT_COMPLETE, SBI_SSE_EVENT_COMPLETE); + DEFINE(NR_CPUS, NR_CPUS); +#endif } diff --git a/arch/riscv/kernel/sse.c b/arch/riscv/kernel/sse.c new file mode 100644 index 00000000000000..b59bda2c1f5869 --- /dev/null +++ b/arch/riscv/kernel/sse.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2024 Rivos Inc. + */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +DEFINE_PER_CPU(struct task_struct *, __sse_entry_task); + +void __weak sse_handle_event(struct sse_event_arch_data *arch_evt, struct pt_regs *regs) +{ +} + +void do_sse(struct sse_event_arch_data *arch_evt, struct pt_regs *regs) +{ + nmi_enter(); + + /* Retrieve missing GPRs from SBI */ + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, arch_evt->evt_id, + SBI_SSE_ATTR_INTERRUPTED_A6, + (SBI_SSE_ATTR_INTERRUPTED_A7 - SBI_SSE_ATTR_INTERRUPTED_A6) + 1, + arch_evt->interrupted_state_phys, 0, 0); + + memcpy(®s->a6, &arch_evt->interrupted, sizeof(arch_evt->interrupted)); + + sse_handle_event(arch_evt, regs); + + /* + * The SSE delivery path does not uses the "standard" exception path and + * thus does not process any pending signal/softirqs. Some drivers might + * enqueue pending work that needs to be handled as soon as possible. + * For that purpose, set the software interrupt pending bit which will + * be serviced once interrupts are reenabled. + */ + if (!user_mode(regs) && !regs_irqs_disabled(regs)) + csr_set(CSR_IP, IE_SIE); + + nmi_exit(); +} + +#ifdef CONFIG_VMAP_STACK +static unsigned long *sse_stack_alloc(unsigned int cpu, unsigned int size) +{ + return arch_alloc_vmap_stack(size, cpu_to_node(cpu)); +} + +static void sse_stack_free(unsigned long *stack) +{ + vfree(stack); +} +#else /* CONFIG_VMAP_STACK */ + +static unsigned long *sse_stack_alloc(unsigned int cpu, unsigned int size) +{ + return kmalloc(size, GFP_KERNEL); +} + +static void sse_stack_free(unsigned long *stack) +{ + kfree(stack); +} + +#endif /* CONFIG_VMAP_STACK */ + +static int sse_init_scs(int cpu, struct sse_event_arch_data *arch_evt) +{ + void *stack; + + if (!scs_is_enabled()) + return 0; + + stack = scs_alloc(cpu_to_node(cpu)); + if (!stack) + return -ENOMEM; + + arch_evt->shadow_stack = stack; + + return 0; +} + +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, int cpu) +{ + void *stack; + + arch_evt->evt_id = evt_id; + stack = sse_stack_alloc(cpu, SSE_STACK_SIZE); + if (!stack) + return -ENOMEM; + + arch_evt->stack = stack + SSE_STACK_SIZE; + + if (sse_init_scs(cpu, arch_evt)) { + sse_stack_free(arch_evt->stack - SSE_STACK_SIZE); + return -ENOMEM; + } + + if (sse_event_is_global(evt_id)) { + arch_evt->interrupted_state_phys = + virt_to_phys(&arch_evt->interrupted); + } else { + arch_evt->interrupted_state_phys = + per_cpu_ptr_to_phys(&arch_evt->interrupted); + } + + return 0; +} + +void arch_sse_free_event(struct sse_event_arch_data *arch_evt) +{ + scs_free(arch_evt->shadow_stack); + sse_stack_free(arch_evt->stack - SSE_STACK_SIZE); +} + +int arch_sse_register_event(struct sse_event_arch_data *arch_evt) +{ + struct sbiret sret; + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_REGISTER, arch_evt->evt_id, + (unsigned long) handle_sse, (unsigned long) arch_evt, + 0, 0, 0); + + return sbi_err_map_linux_errno(sret.error); +} diff --git a/arch/riscv/kernel/sse_entry.S b/arch/riscv/kernel/sse_entry.S new file mode 100644 index 00000000000000..c860fc4f36c508 --- /dev/null +++ b/arch/riscv/kernel/sse_entry.S @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Rivos Inc. + */ + +#include +#include + +#include +#include +#include + +/* When entering handle_sse, the following registers are set: + * a6: contains the hartid + * a7: contains struct sse_registered_event pointer + */ +SYM_CODE_START(handle_sse) + /* Save stack temporarily */ + REG_S sp, SSE_REG_EVT_TMP(a7) + /* Set entry stack */ + REG_L sp, SSE_REG_EVT_STACK(a7) + + addi sp, sp, -(PT_SIZE_ON_STACK) + REG_S ra, PT_RA(sp) + REG_S s0, PT_S0(sp) + REG_S s1, PT_S1(sp) + REG_S s2, PT_S2(sp) + REG_S s3, PT_S3(sp) + REG_S s4, PT_S4(sp) + REG_S s5, PT_S5(sp) + REG_S s6, PT_S6(sp) + REG_S s7, PT_S7(sp) + REG_S s8, PT_S8(sp) + REG_S s9, PT_S9(sp) + REG_S s10, PT_S10(sp) + REG_S s11, PT_S11(sp) + REG_S tp, PT_TP(sp) + REG_S t0, PT_T0(sp) + REG_S t1, PT_T1(sp) + REG_S t2, PT_T2(sp) + REG_S t3, PT_T3(sp) + REG_S t4, PT_T4(sp) + REG_S t5, PT_T5(sp) + REG_S t6, PT_T6(sp) + REG_S gp, PT_GP(sp) + REG_S a0, PT_A0(sp) + REG_S a1, PT_A1(sp) + REG_S a2, PT_A2(sp) + REG_S a3, PT_A3(sp) + REG_S a4, PT_A4(sp) + REG_S a5, PT_A5(sp) + + /* Retrieve entry sp */ + REG_L a4, SSE_REG_EVT_TMP(a7) + /* Save CSRs */ + csrr a0, CSR_EPC + csrr a1, CSR_SSTATUS + csrr a2, CSR_STVAL + csrr a3, CSR_SCAUSE + + REG_S a0, PT_EPC(sp) + REG_S a1, PT_STATUS(sp) + REG_S a2, PT_BADADDR(sp) + REG_S a3, PT_CAUSE(sp) + REG_S a4, PT_SP(sp) + + /* Disable user memory access and floating/vector computing */ + li t0, SR_SUM | SR_FS_VS + csrc CSR_STATUS, t0 + + load_global_pointer + scs_load_sse_stack a7 + + /* Restore current task struct from __sse_entry_task */ + li t1, NR_CPUS + move t3, zero + +#ifdef CONFIG_SMP + /* Find the CPU id associated to the hart id */ + la t0, __cpuid_to_hartid_map +.Lhart_id_loop: + REG_L t2, 0(t0) + beq t2, a6, .Lcpu_id_found + + /* Increment pointer and CPU number */ + addi t3, t3, 1 + addi t0, t0, RISCV_SZPTR + bltu t3, t1, .Lhart_id_loop + + /* + * This should never happen since we expect the hart_id to match one + * of our CPU, but better be safe than sorry + */ + la tp, init_task + la a0, sse_hart_id_panic_string + la t0, panic + jalr t0 + +.Lcpu_id_found: +#endif + asm_per_cpu_with_cpu t2 __sse_entry_task t1 t3 + REG_L tp, 0(t2) + + move a1, sp /* pt_regs on stack */ + + /* + * Save sscratch for restoration since we might have interrupted the + * kernel in early exception path and thus, we don't know the content of + * sscratch. + */ + csrr s4, CSR_SSCRATCH + /* In-kernel scratch is 0 */ + csrw CSR_SCRATCH, x0 + + move a0, a7 + + call do_sse + + csrw CSR_SSCRATCH, s4 + + REG_L a0, PT_EPC(sp) + REG_L a1, PT_STATUS(sp) + REG_L a2, PT_BADADDR(sp) + REG_L a3, PT_CAUSE(sp) + csrw CSR_EPC, a0 + csrw CSR_SSTATUS, a1 + csrw CSR_STVAL, a2 + csrw CSR_SCAUSE, a3 + + REG_L ra, PT_RA(sp) + REG_L s0, PT_S0(sp) + REG_L s1, PT_S1(sp) + REG_L s2, PT_S2(sp) + REG_L s3, PT_S3(sp) + REG_L s4, PT_S4(sp) + REG_L s5, PT_S5(sp) + REG_L s6, PT_S6(sp) + REG_L s7, PT_S7(sp) + REG_L s8, PT_S8(sp) + REG_L s9, PT_S9(sp) + REG_L s10, PT_S10(sp) + REG_L s11, PT_S11(sp) + REG_L tp, PT_TP(sp) + REG_L t0, PT_T0(sp) + REG_L t1, PT_T1(sp) + REG_L t2, PT_T2(sp) + REG_L t3, PT_T3(sp) + REG_L t4, PT_T4(sp) + REG_L t5, PT_T5(sp) + REG_L t6, PT_T6(sp) + REG_L gp, PT_GP(sp) + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + + REG_L sp, PT_SP(sp) + + li a7, SBI_EXT_SSE + li a6, SBI_SSE_EVENT_COMPLETE + ecall + +SYM_CODE_END(handle_sse) + +SYM_DATA_START_LOCAL(sse_hart_id_panic_string) + .ascii "Unable to match hart_id with cpu\0" +SYM_DATA_END(sse_hart_id_panic_string) diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index aadc395ee16813..fe72e705067cea 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -278,6 +278,7 @@ source "drivers/firmware/meson/Kconfig" source "drivers/firmware/microchip/Kconfig" source "drivers/firmware/psci/Kconfig" source "drivers/firmware/qcom/Kconfig" +source "drivers/firmware/riscv/Kconfig" source "drivers/firmware/samsung/Kconfig" source "drivers/firmware/smccc/Kconfig" source "drivers/firmware/tegra/Kconfig" diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 4ddec2820c96fa..6cdd84570ea731 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -34,6 +34,7 @@ obj-y += efi/ obj-y += imx/ obj-y += psci/ obj-y += qcom/ +obj-y += riscv/ obj-y += samsung/ obj-y += smccc/ obj-y += tegra/ diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig new file mode 100644 index 00000000000000..8056ed3262d912 --- /dev/null +++ b/drivers/firmware/riscv/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Risc-V Specific firmware drivers" +depends on RISCV + +config RISCV_SSE + bool "Enable SBI Supervisor Software Events support" + depends on RISCV_SBI + default y + help + The Supervisor Software Events support allow the SBI to deliver + NMI-like notifications to the supervisor mode software. When enable, + this option provides support to register callbacks on specific SSE + events. + +endmenu diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile new file mode 100644 index 00000000000000..4ccfcbbc28ea3e --- /dev/null +++ b/drivers/firmware/riscv/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RISCV_SSE) += riscv_sse.o diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sse.c new file mode 100644 index 00000000000000..05e4bc8dfa9914 --- /dev/null +++ b/drivers/firmware/riscv/riscv_sse.c @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Rivos Inc. + */ + +#define pr_fmt(fmt) "sse: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct sse_event { + struct list_head list; + u32 evt_id; + u32 priority; + sse_event_handler *handler; + void *handler_arg; + /* Only valid for global events */ + unsigned int cpu; + + union { + struct sse_registered_event *global; + struct sse_registered_event __percpu *local; + }; +}; + +static int sse_hp_state; +static bool sse_available; +static DEFINE_SPINLOCK(events_list_lock); +static LIST_HEAD(events); +static DEFINE_MUTEX(sse_mutex); + +struct sse_registered_event { + struct sse_event_arch_data arch; + struct sse_event *event; + unsigned long attr_buf; + bool is_enabled; +}; + +void sse_handle_event(struct sse_event_arch_data *arch_event, + struct pt_regs *regs) +{ + int ret; + struct sse_registered_event *reg_evt = + container_of(arch_event, struct sse_registered_event, arch); + struct sse_event *evt = reg_evt->event; + + ret = evt->handler(evt->evt_id, evt->handler_arg, regs); + if (ret) + pr_warn("event %x handler failed with error %d\n", evt->evt_id, + ret); +} + +static struct sse_event *sse_event_get(u32 evt) +{ + struct sse_event *event = NULL, *tmp; + + scoped_guard(spinlock, &events_list_lock) { + list_for_each_entry(tmp, &events, list) { + if (tmp->evt_id == evt) + return event; + } + } + + return NULL; +} + +static phys_addr_t sse_event_get_phys(struct sse_registered_event *reg_evt, + void *addr) +{ + phys_addr_t phys; + + if (sse_event_is_global(reg_evt->event->evt_id)) + phys = virt_to_phys(addr); + else + phys = per_cpu_ptr_to_phys(addr); + + return phys; +} + +static struct sse_registered_event *sse_get_reg_evt(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return event->global; + else + return per_cpu_ptr(event->local, smp_processor_id()); +} + +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) +{ + struct sbiret ret; + u32 evt = event->evt_id; + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); + if (ret.error) + pr_debug("Failed to execute func %lx, event %x, error %ld\n", + func, evt, ret.error); + + if (func == SBI_SSE_EVENT_DISABLE) + reg_evt->is_enabled = false; + else if (func == SBI_SSE_EVENT_ENABLE) + reg_evt->is_enabled = true; + + return sbi_err_map_linux_errno(ret.error); +} + +int sse_event_disable_local(struct sse_event *event) +{ + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); +} + +int sse_event_enable_local(struct sse_event *event) +{ + return sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); +} + +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long *val) +{ + struct sbiret sret; + u32 evt = reg_evt->event->evt_id; + unsigned long phys; + + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, + attr_id, 1, phys, 0, 0); + if (sret.error) { + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + return sbi_err_map_linux_errno(sret.error); + } + + *val = reg_evt->attr_buf; + + return 0; +} + +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long val) +{ + struct sbiret sret; + u32 evt = reg_evt->event->evt_id; + unsigned long phys; + + reg_evt->attr_buf = val; + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, + attr_id, 1, phys, 0, 0); + if (sret.error) { + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + return sbi_err_map_linux_errno(sret.error); + } + + return 0; +} + +static int sse_event_set_target_cpu_nolock(struct sse_event *event, + unsigned int cpu) +{ + unsigned int hart_id = cpuid_to_hartid_map(cpu); + struct sse_registered_event *reg_evt = event->global; + u32 evt = event->evt_id; + bool was_enabled; + int ret; + + if (!sse_event_is_global(evt)) + return -EINVAL; + + was_enabled = reg_evt->is_enabled; + if (was_enabled) + sse_event_disable_local(event); + do { + ret = sse_event_attr_set_nolock(reg_evt, + SBI_SSE_ATTR_PREFERRED_HART, + hart_id); + } while (ret == -EINVAL); + + if (ret == 0) + event->cpu = cpu; + + if (was_enabled) + sse_event_enable_local(event); + + return 0; +} + +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) +{ + int ret; + + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + + if (!cpu_online(cpu)) + return -EINVAL; + + ret = sse_event_set_target_cpu_nolock(event, cpu); + + cpus_read_unlock(); + } + + return ret; +} + +static int sse_event_init_registered(unsigned int cpu, + struct sse_registered_event *reg_evt, + struct sse_event *event) +{ + reg_evt->event = event; + arch_sse_init_event(®_evt->arch, event->evt_id, cpu); + + return 0; +} + +static void sse_event_free_registered(struct sse_registered_event *reg_evt) +{ + arch_sse_free_event(®_evt->arch); +} + +static int sse_event_alloc_global(struct sse_event *event) +{ + int err; + struct sse_registered_event *reg_evt; + + reg_evt = kzalloc(sizeof(*reg_evt), GFP_KERNEL); + if (!reg_evt) + return -ENOMEM; + + event->global = reg_evt; + err = sse_event_init_registered(smp_processor_id(), reg_evt, event); + if (err) + kfree(reg_evt); + + return err; +} + +static int sse_event_alloc_local(struct sse_event *event) +{ + int err; + unsigned int cpu, err_cpu; + struct sse_registered_event *reg_evt; + struct sse_registered_event __percpu *reg_evts; + + reg_evts = alloc_percpu(struct sse_registered_event); + if (!reg_evts) + return -ENOMEM; + + event->local = reg_evts; + + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(reg_evts, cpu); + err = sse_event_init_registered(cpu, reg_evt, event); + if (err) { + err_cpu = cpu; + goto err_free_per_cpu; + } + } + + return 0; + +err_free_per_cpu: + for_each_possible_cpu(cpu) { + if (cpu == err_cpu) + break; + reg_evt = per_cpu_ptr(reg_evts, cpu); + sse_event_free_registered(reg_evt); + } + + free_percpu(reg_evts); + + return err; +} + +static struct sse_event *sse_event_alloc(u32 evt, u32 priority, + sse_event_handler *handler, void *arg) +{ + int err; + struct sse_event *event; + + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return ERR_PTR(-ENOMEM); + + event->evt_id = evt; + event->priority = priority; + event->handler_arg = arg; + event->handler = handler; + + if (sse_event_is_global(evt)) { + err = sse_event_alloc_global(event); + if (err) + goto err_alloc_reg_evt; + } else { + err = sse_event_alloc_local(event); + if (err) + goto err_alloc_reg_evt; + } + + return event; + +err_alloc_reg_evt: + kfree(event); + + return ERR_PTR(err); +} + +static int sse_sbi_register_event(struct sse_event *event, + struct sse_registered_event *reg_evt) +{ + int ret; + + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, + event->priority); + if (ret) + return ret; + + return arch_sse_register_event(®_evt->arch); +} + +static int sse_event_register_local(struct sse_event *event) +{ + int ret; + struct sse_registered_event *reg_evt = per_cpu_ptr(event->local, + smp_processor_id()); + + ret = sse_sbi_register_event(event, reg_evt); + if (ret) + pr_debug("Failed to register event %x: err %d\n", event->evt_id, + ret); + + return ret; +} + +static int sse_sbi_unregister_event(struct sse_event *event) +{ + return sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); +} + +struct sse_per_cpu_evt { + struct sse_event *event; + unsigned long func; + atomic_t error; +}; + +static void sse_event_per_cpu_func(void *info) +{ + int ret; + struct sse_per_cpu_evt *cpu_evt = info; + + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) + ret = sse_event_register_local(cpu_evt->event); + else + ret = sse_sbi_event_func(cpu_evt->event, cpu_evt->func); + + if (ret) + atomic_set(&cpu_evt->error, ret); +} + +static void sse_event_free(struct sse_event *event) +{ + unsigned int cpu; + struct sse_registered_event *reg_evt; + + if (sse_event_is_global(event->evt_id)) { + sse_event_free_registered(event->global); + kfree(event->global); + } else { + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(event->local, cpu); + sse_event_free_registered(reg_evt); + } + free_percpu(event->local); + } + + kfree(event); +} + +int sse_event_enable(struct sse_event *event) +{ + int ret = 0, cpu; + struct sse_per_cpu_evt cpu_evt; + struct sse_registered_event *reg_evt; + + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + if (sse_event_is_global(event->evt_id)) { + reg_evt = event->global; + ret = sse_event_enable_local(event); + } else { + cpu_evt.event = event; + atomic_set(&cpu_evt.error, 0); + cpu_evt.func = SBI_SSE_EVENT_ENABLE; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + ret = atomic_read(&cpu_evt.error); + if (ret) { + cpu_evt.func = SBI_SSE_EVENT_DISABLE; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + } else { + for_each_online_cpu(cpu) { + + } + } + } + cpus_read_unlock(); + } + + return ret; +} + +static void sse_events_mask(void) +{ + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_MASK, 0, 0, 0, 0, 0, 0); +} + +static void sse_events_unmask(void) +{ + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_UNMASK, 0, 0, 0, 0, 0, 0); +} + +static void sse_event_disable_nolock(struct sse_event *event) +{ + struct sse_per_cpu_evt cpu_evt; + + if (sse_event_is_global(event->evt_id)) { + sse_event_disable_local(event); + } else { + cpu_evt.event = event; + cpu_evt.func = SBI_SSE_EVENT_DISABLE; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + } +} + +void sse_event_disable(struct sse_event *event) +{ + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + sse_event_disable_nolock(event); + cpus_read_unlock(); + } +} + +struct sse_event *sse_event_register(u32 evt, u32 priority, + sse_event_handler *handler, void *arg) +{ + struct sse_per_cpu_evt cpu_evt; + struct sse_event *event; + int ret = 0; + + if (!sse_available) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&sse_mutex); + if (sse_event_get(evt)) { + pr_debug("Event %x already registered\n", evt); + ret = -EEXIST; + goto out_unlock; + } + + event = sse_event_alloc(evt, priority, handler, arg); + if (IS_ERR(event)) { + ret = PTR_ERR(event); + goto out_unlock; + } + + cpus_read_lock(); + if (sse_event_is_global(evt)) { + unsigned long preferred_hart; + + ret = sse_event_attr_get_no_lock(event->global, + SBI_SSE_ATTR_PREFERRED_HART, + &preferred_hart); + if (ret) + goto err_event_free; + event->cpu = riscv_hartid_to_cpuid(preferred_hart); + + ret = sse_sbi_register_event(event, event->global); + if (ret) + goto err_event_free; + + } else { + cpu_evt.event = event; + atomic_set(&cpu_evt.error, 0); + cpu_evt.func = SBI_SSE_EVENT_REGISTER; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + ret = atomic_read(&cpu_evt.error); + if (ret) { + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + goto err_event_free; + } + } + cpus_read_unlock(); + + scoped_guard(spinlock, &events_list_lock) + list_add(&event->list, &events); + + mutex_unlock(&sse_mutex); + + return event; + +err_event_free: + cpus_read_unlock(); + sse_event_free(event); +out_unlock: + mutex_unlock(&sse_mutex); + + return ERR_PTR(ret); +} + +static void sse_event_unregister_nolock(struct sse_event *event) +{ + struct sse_per_cpu_evt cpu_evt; + + if (sse_event_is_global(event->evt_id)) { + sse_sbi_unregister_event(event); + } else { + cpu_evt.event = event; + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + } +} + +void sse_event_unregister(struct sse_event *event) +{ + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + sse_event_unregister_nolock(event); + cpus_read_unlock(); + + scoped_guard(spinlock, &events_list_lock) + list_del(&event->list); + + sse_event_free(event); + } +} + +static int sse_cpu_online(unsigned int cpu) +{ + struct sse_event *event; + + scoped_guard(spinlock, &events_list_lock) { + list_for_each_entry(event, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + + sse_event_register_local(event); + if (sse_get_reg_evt(event)) + sse_event_enable_local(event); + } + } + + /* Ready to handle events. Unmask SSE. */ + sse_events_unmask(); + + return 0; +} + +static int sse_cpu_teardown(unsigned int cpu) +{ + unsigned int next_cpu; + struct sse_event *event; + + /* Mask the sse events */ + sse_events_mask(); + + scoped_guard(spinlock, &events_list_lock) { + list_for_each_entry(event, &events, list) { + if (!sse_event_is_global(event->evt_id)) { + + if (event->global->is_enabled) + sse_event_disable_local(event); + + sse_sbi_unregister_event(event); + continue; + } + + if (event->cpu != smp_processor_id()) + continue; + + /* Update destination hart for global event */ + next_cpu = cpumask_any_but(cpu_online_mask, cpu); + sse_event_set_target_cpu_nolock(event, next_cpu); + } + } + + return 0; +} + +static void sse_reset(void) +{ + struct sse_event *event = NULL; + + list_for_each_entry(event, &events, list) { + sse_event_disable_nolock(event); + sse_event_unregister_nolock(event); + } +} + +static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + WARN_ON_ONCE(preemptible()); + + switch (action) { + case CPU_PM_ENTER: + sse_events_mask(); + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + sse_events_unmask(); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block sse_pm_nb = { + .notifier_call = sse_pm_notifier, +}; + +/* + * Mask all CPUs and unregister all events on panic, reboot or kexec. + */ +static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + cpuhp_remove_state(sse_hp_state); + + sse_reset(); + + return NOTIFY_OK; +} + +static struct notifier_block sse_reboot_nb = { + .notifier_call = sse_reboot_notifier, +}; + +static int __init sse_init(void) +{ + int cpu, ret; + + if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { + pr_err("Missing SBI SSE extension\n"); + return -EOPNOTSUPP; + } + pr_info("SBI SSE extension detected\n"); + + for_each_possible_cpu(cpu) + INIT_LIST_HEAD(&events); + + ret = cpu_pm_register_notifier(&sse_pm_nb); + if (ret) { + pr_warn("Failed to register CPU PM notifier...\n"); + return ret; + } + + ret = register_reboot_notifier(&sse_reboot_nb); + if (ret) { + pr_warn("Failed to register reboot notifier...\n"); + goto remove_cpupm; + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/sse:online", + sse_cpu_online, sse_cpu_teardown); + if (ret < 0) + goto remove_reboot; + + sse_hp_state = ret; + sse_available = true; + + return 0; + +remove_reboot: + unregister_reboot_notifier(&sse_reboot_nb); + +remove_cpupm: + cpu_pm_unregister_notifier(&sse_pm_nb); + + return ret; +} +arch_initcall(sse_init); diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index 4e268de351c48a..3462f3a21e5fd5 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -105,6 +105,15 @@ config RISCV_PMU_SBI full perf feature support i.e. counter overflow, privilege mode filtering, counter configuration. +config RISCV_PMU_SSE + depends on RISCV_PMU && RISCV_SSE + bool "RISC-V PMU SSE events" + default n + help + Say y if you want to use SSE events to deliver PMU interrupts. This + provides a way to profile the kernel at any level by using NMI-like + SSE events. + config STARFIVE_STARLINK_PMU depends on ARCH_STARFIVE || COMPILE_TEST depends on 64BIT diff --git a/drivers/perf/riscv_pmu.c b/drivers/perf/riscv_pmu.c index 7644147d50b46a..1eb28381b80f51 100644 --- a/drivers/perf/riscv_pmu.c +++ b/drivers/perf/riscv_pmu.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -254,6 +255,22 @@ void riscv_pmu_start(struct perf_event *event, int flags) perf_event_update_userpage(event); } +static void riscv_pmu_disable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + + if (rvpmu->sse_evt) + sse_event_disable_local(rvpmu->sse_evt); +} + +static void riscv_pmu_enable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + + if (rvpmu->sse_evt) + sse_event_enable_local(rvpmu->sse_evt); +} + static int riscv_pmu_add(struct perf_event *event, int flags) { struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); @@ -411,6 +428,8 @@ struct riscv_pmu *riscv_pmu_alloc(void) .event_mapped = riscv_pmu_event_mapped, .event_unmapped = riscv_pmu_event_unmapped, .event_idx = riscv_pmu_event_idx, + .pmu_enable = riscv_pmu_enable, + .pmu_disable = riscv_pmu_disable, .add = riscv_pmu_add, .del = riscv_pmu_del, .start = riscv_pmu_start, diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 698de8ddf895ba..885d04a2c338ff 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -948,10 +949,10 @@ static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu, pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask); } -static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) +static irqreturn_t pmu_sbi_ovf_handler(struct cpu_hw_events *cpu_hw_evt, + struct pt_regs *regs, bool from_sse) { struct perf_sample_data data; - struct pt_regs *regs; struct hw_perf_event *hw_evt; union sbi_pmu_ctr_info *info; int lidx, hidx, fidx; @@ -959,7 +960,6 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) struct perf_event *event; u64 overflow; u64 overflowed_ctrs = 0; - struct cpu_hw_events *cpu_hw_evt = dev; u64 start_clock = sched_clock(); struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; @@ -969,13 +969,15 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) /* Firmware counter don't support overflow yet */ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); if (fidx == RISCV_MAX_COUNTERS) { - csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); + if (!from_sse) + csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); return IRQ_NONE; } event = cpu_hw_evt->events[fidx]; if (!event) { - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (!from_sse) + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); return IRQ_NONE; } @@ -990,16 +992,16 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) /* * Overflow interrupt pending bit should only be cleared after stopping - * all the counters to avoid any race condition. + * all the counters to avoid any race condition. When using SSE, + * interrupt is cleared when stopping counters. */ - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (!from_sse) + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); /* No overflow bit is set */ if (!overflow) return IRQ_NONE; - regs = get_irq_regs(); - for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { struct perf_event *event = cpu_hw_evt->events[lidx]; @@ -1055,6 +1057,52 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) return IRQ_HANDLED; } +static irqreturn_t pmu_sbi_ovf_irq_handler(int irq, void *dev) +{ + return pmu_sbi_ovf_handler(dev, get_irq_regs(), false); +} + +#ifdef CONFIG_RISCV_PMU_SSE +static int pmu_sbi_ovf_sse_handler(uint32_t evt, void *arg, + struct pt_regs *regs) +{ + struct cpu_hw_events __percpu *hw_events = arg; + struct cpu_hw_events *hw_event = raw_cpu_ptr(hw_events); + + pmu_sbi_ovf_handler(hw_event, regs, true); + + return 0; +} + +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) +{ + int ret; + struct sse_event *evt; + struct cpu_hw_events __percpu *hw_events = pmu->hw_events; + + evt = sse_event_register(SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, 0, + pmu_sbi_ovf_sse_handler, hw_events); + if (IS_ERR(evt)) + return PTR_ERR(evt); + + ret = sse_event_enable(evt); + if (ret) { + sse_event_unregister(evt); + return ret; + } + + pr_info("using SSE for PMU event delivery\n"); + pmu->sse_evt = evt; + + return ret; +} +#else +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) +{ + return -EOPNOTSUPP; +} +#endif + static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) { struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); @@ -1105,6 +1153,10 @@ static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pde struct cpu_hw_events __percpu *hw_events = pmu->hw_events; struct irq_domain *domain = NULL; + ret = pmu_sbi_setup_sse(pmu); + if (!ret) + return 0; + if (riscv_isa_extension_available(NULL, SSCOFPMF)) { riscv_pmu_irq_num = RV_IRQ_PMU; riscv_pmu_use_irq = true; @@ -1139,7 +1191,7 @@ static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pde return -ENODEV; } - ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events); + ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_irq_handler, "riscv-pmu", hw_events); if (ret) { pr_err("registering percpu irq failed [%d]\n", ret); return ret; diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index 701974639ff2ca..d4a5c55fe077cb 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -28,6 +28,8 @@ #define RISCV_PMU_CONFIG1_GUEST_EVENTS 0x1 +struct sse_event; + struct cpu_hw_events { /* currently enabled events */ int n_events; @@ -54,6 +56,7 @@ struct riscv_pmu { char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); + struct sse_event *sse_evt; unsigned long cmask; u64 (*ctr_read)(struct perf_event *event); diff --git a/include/linux/riscv_sse.h b/include/linux/riscv_sse.h new file mode 100644 index 00000000000000..8653fa74ec8296 --- /dev/null +++ b/include/linux/riscv_sse.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Rivos Inc. + */ + +#ifndef __LINUX_RISCV_SSE_H +#define __LINUX_RISCV_SSE_H + +#include +#include + +struct sse_event; +struct pt_regs; + +typedef int (sse_event_handler)(u32 event_num, void *arg, struct pt_regs *regs); + +#ifdef CONFIG_RISCV_SSE + +struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler *handler, void *arg); + +void sse_event_unregister(struct sse_event *evt); + +int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); + +int sse_event_enable(struct sse_event *sse_evt); + +void sse_event_disable(struct sse_event *sse_evt); + +int sse_event_enable_local(struct sse_event *sse_evt); +int sse_event_disable_local(struct sse_event *sse_evt); + +#else +static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler *handler, + void *arg) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void sse_event_unregister(struct sse_event *evt) {} + +static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, + unsigned int cpu) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_enable(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline void sse_event_disable(struct sse_event *sse_evt) {} + + +#endif + +#endif /* __LINUX_RISCV_SSE_H */