diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index c6ec9290fe07f0..1f3853a6876039 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -232,6 +232,12 @@ properties: ratified at commit d70011dde6c2 ("Update to ratified state") of riscv-j-extension. + - const: ssqosid + description: | + The Ssqosid extension for Quality of Service ID is ratified + as v1.0 in commit 5059e0ca641c ("Merge pull request #7 from + ved-rivos/Ratified") of riscv-ssqosid. + - const: ssstateen description: | The standard Ssstateen extension for supervisor-mode view of the diff --git a/MAINTAINERS b/MAINTAINERS index 7d10988cbc62b7..133a96a6188b9e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22769,6 +22769,17 @@ F: drivers/perf/riscv_pmu.c F: drivers/perf/riscv_pmu_legacy.c F: drivers/perf/riscv_pmu_sbi.c +RISC-V QOS RESCTRL SUPPORT +M: Drew Fustini +R: yunhui cui +L: linux-riscv@lists.infradead.org +S: Supported +F: arch/riscv/include/asm/qos.h +F: arch/riscv/include/asm/resctrl.h +F: arch/riscv/kernel/qos/ +F: drivers/acpi/riscv/rqsc.c +F: include/linux/riscv_qos.h + RISC-V RPMI AND MPXY DRIVERS M: Rahul Pathak M: Anup Patel diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 90c531e6abf5cf..b2fef15b3d4f25 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -595,6 +595,26 @@ config RISCV_ISA_SVNAPOT If you don't know what to do here, say Y. +config RISCV_ISA_SSQOSID + bool "Ssqosid extension support for supervisor mode Quality of Service ID" + depends on MISC_FILESYSTEMS + default n + select ARCH_HAS_CPU_RESCTRL + select RESCTRL_FS + help + Adds support for the Ssqosid ISA extension (Supervisor-mode + Quality of Service ID). + + Ssqosid defines the srmcfg CSR which allows the system to tag the + running process with an RCID (Resource Control ID) and MCID + (Monitoring Counter ID). The RCID is used to determine resource + allocation. The MCID is used to track resource usage in event + counters. + + For example, a cache controller may use the RCID to apply a + cache partitioning scheme and use the MCID to track how much + cache a process, or a group of processes, is using. + config RISCV_ISA_SVPBMT bool "Svpbmt extension support for supervisor mode page-based memory types" depends on 64BIT && MMU diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index 6e13695120bc21..62296a2a519b8a 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -71,6 +71,16 @@ int acpi_get_riscv_isa(struct acpi_table_header *table, void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size, u32 *cboz_size, u32 *cbop_size); + +#ifdef CONFIG_RISCV_ISA_SSQOSID +int __init acpi_parse_rqsc(struct acpi_table_header *table); +#else +static inline int acpi_parse_rqsc(struct acpi_table_header *table) +{ + return -EINVAL; +} +#endif /* CONFIG_RISCV_ISA_SSQOSID */ + #else static inline void acpi_init_rintc_map(void) { } static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 31b8988f4488da..dc13835bd6612b 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -84,6 +84,11 @@ #define SATP_ASID_MASK _AC(0xFFFF, UL) #endif +/* SRMCFG fields */ +#define SRMCFG_RCID_MASK _AC(0x00000FFF, UL) +#define SRMCFG_MCID_MASK _AC(0x00000FFF, UL) +#define SRMCFG_MCID_SHIFT 16 + /* Exception cause high bit - is an interrupt if set */ #define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) @@ -328,6 +333,7 @@ #define CSR_STVAL 0x143 #define CSR_SIP 0x144 #define CSR_SATP 0x180 +#define CSR_SRMCFG 0x181 #define CSR_STIMECMP 0x14D #define CSR_STIMECMPH 0x15D diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 7ef8e5f55c8dcf..b83dae5cebb992 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -112,6 +112,7 @@ #define RISCV_ISA_EXT_ZCLSD 103 #define RISCV_ISA_EXT_ZICFILP 104 #define RISCV_ISA_EXT_ZICFISS 105 +#define RISCV_ISA_EXT_SSQOSID 106 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 4c3dd94d0f6384..feaee2502593dd 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -123,6 +123,9 @@ struct thread_struct { /* A forced icache flush is not needed if migrating to the previous cpu. */ unsigned int prev_cpu; #endif +#ifdef CONFIG_RISCV_ISA_SSQOSID + u32 srmcfg; +#endif }; /* Whitelist the fstate from the task_struct for hardened usercopy */ diff --git a/arch/riscv/include/asm/qos.h b/arch/riscv/include/asm/qos.h new file mode 100644 index 00000000000000..aa96bb5a153ca3 --- /dev/null +++ b/arch/riscv/include/asm/qos.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_QOS_H +#define _ASM_RISCV_QOS_H + +#ifdef CONFIG_RISCV_ISA_SSQOSID + +#include + +#include +#include + +/* cached value of srmcfg csr for each cpu */ +DECLARE_PER_CPU(u32, cpu_srmcfg); + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DECLARE_PER_CPU(u32, cpu_srmcfg_default); + +static inline void __switch_to_srmcfg(struct task_struct *next) +{ + u32 thread_srmcfg; + + thread_srmcfg = READ_ONCE(next->thread.srmcfg); + + /* + * Tasks in the default resource group have closid=0 and rmid=0, + * so thread.srmcfg is 0. For these tasks, use this CPU's default + * srmcfg instead. This implements resctrl rule 2: a default-group + * task running on a CPU assigned to a specific group uses that + * group's allocations. + */ + if (thread_srmcfg == 0) + thread_srmcfg = __this_cpu_read(cpu_srmcfg_default); + + if (thread_srmcfg != __this_cpu_read(cpu_srmcfg)) { + __this_cpu_write(cpu_srmcfg, thread_srmcfg); + csr_write(CSR_SRMCFG, thread_srmcfg); + } +} + +static __always_inline bool has_srmcfg(void) +{ + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID); +} + +#else /* ! CONFIG_RISCV_ISA_SSQOSID */ + +struct task_struct; +static __always_inline bool has_srmcfg(void) { return false; } +static inline void __switch_to_srmcfg(struct task_struct *next) { } + +#endif /* CONFIG_RISCV_ISA_SSQOSID */ +#endif /* _ASM_RISCV_QOS_H */ diff --git a/arch/riscv/include/asm/resctrl.h b/arch/riscv/include/asm/resctrl.h new file mode 100644 index 00000000000000..48ad45cbe25c1e --- /dev/null +++ b/arch/riscv/include/asm/resctrl.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_RISCV_RESCTRL_H +#define _ASM_RISCV_RESCTRL_H + +#include + +#endif /* _ASM_RISCV_RESCTRL_H */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920ca..1c7ea53ec012ad 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef CONFIG_FPU extern void __fstate_save(struct task_struct *save_to); @@ -119,6 +120,8 @@ do { \ __switch_to_fpu(__prev, __next); \ if (has_vector() || has_xtheadvector()) \ __switch_to_vector(__prev, __next); \ + if (has_srmcfg()) \ + __switch_to_srmcfg(__next); \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index cabb99cadfb6d1..82157aae640156 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -128,3 +128,5 @@ obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o + +obj-$(CONFIG_RISCV_ISA_SSQOSID) += qos/ diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 1734f9a4c2fd70..c0717a861a3c80 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -582,6 +582,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), + __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), diff --git a/arch/riscv/kernel/qos/Makefile b/arch/riscv/kernel/qos/Makefile new file mode 100644 index 00000000000000..9ed0c13a854d9b --- /dev/null +++ b/arch/riscv/kernel/qos/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_RISCV_ISA_SSQOSID) += qos.o qos_resctrl.o diff --git a/arch/riscv/kernel/qos/internal.h b/arch/riscv/kernel/qos/internal.h new file mode 100644 index 00000000000000..edbcbd9471b194 --- /dev/null +++ b/arch/riscv/kernel/qos/internal.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_RISCV_QOS_INTERNAL_H +#define _ASM_RISCV_QOS_INTERNAL_H + +#include +#include +#include + +#define RISCV_RESCTRL_EMPTY_CLOSID ((u32)~0) + +#define CBQRI_CC_CAPABILITIES_OFF 0 +#define CBQRI_CC_MON_CTL_OFF 8 +#define CBQRI_CC_MON_CTL_VAL_OFF 16 +#define CBQRI_CC_ALLOC_CTL_OFF 24 +#define CBQRI_CC_BLOCK_MASK_OFF 32 + +#define CBQRI_BC_CAPABILITIES_OFF 0 +#define CBQRI_BC_MON_CTL_OFF 8 +#define CBQRI_BC_MON_CTR_VAL_OFF 16 +#define CBQRI_BC_ALLOC_CTL_OFF 24 +#define CBQRI_BC_BW_ALLOC_OFF 32 + +#define CBQRI_CC_CAPABILITIES_VER_MINOR_MASK GENMASK(3, 0) +#define CBQRI_CC_CAPABILITIES_VER_MAJOR_MASK GENMASK(7, 4) + +#define CBQRI_CC_CAPABILITIES_NCBLKS_MASK GENMASK(23, 8) +#define CBQRI_CC_CAPABILITIES_FRCID_MASK GENMASK(24, 24) + +#define CBQRI_BC_CAPABILITIES_VER_MINOR_MASK GENMASK(3, 0) +#define CBQRI_BC_CAPABILITIES_VER_MAJOR_MASK GENMASK(7, 4) + +#define CBQRI_BC_CAPABILITIES_NBWBLKS_MASK GENMASK(23, 8) +#define CBQRI_BC_CAPABILITIES_MRBWB_MASK GENMASK_ULL(47, 32) + +#define CBQRI_CONTROL_REGISTERS_OP_MASK GENMASK(4, 0) +#define CBQRI_CONTROL_REGISTERS_AT_MASK GENMASK(7, 5) +#define CBQRI_CONTROL_REGISTERS_AT_DATA 0 +#define CBQRI_CONTROL_REGISTERS_AT_CODE 1 +#define CBQRI_CONTROL_REGISTERS_RCID_MASK GENMASK(19, 8) +#define CBQRI_CONTROL_REGISTERS_STATUS_MASK GENMASK_ULL(38, 32) +#define CBQRI_CONTROL_REGISTERS_BUSY_MASK GENMASK_ULL(39, 39) +#define CBQRI_CONTROL_REGISTERS_RBWB_MASK GENMASK(15, 0) + +#define CBQRI_CC_MON_CTL_OP_CONFIG_EVENT 1 +#define CBQRI_CC_MON_CTL_OP_READ_COUNTER 2 +#define CBQRI_CC_MON_CTL_STATUS_SUCCESS 1 + +#define CBQRI_CC_ALLOC_CTL_OP_CONFIG_LIMIT 1 +#define CBQRI_CC_ALLOC_CTL_OP_READ_LIMIT 2 +#define CBQRI_CC_ALLOC_CTL_OP_FLUSH_RCID 3 +#define CBQRI_CC_ALLOC_CTL_STATUS_SUCCESS 1 + +#define CBQRI_BC_MON_CTL_OP_CONFIG_EVENT 1 +#define CBQRI_BC_MON_CTL_OP_READ_COUNTER 2 +#define CBQRI_BC_MON_CTL_STATUS_SUCCESS 1 + +#define CBQRI_BC_ALLOC_CTL_OP_CONFIG_LIMIT 1 +#define CBQRI_BC_ALLOC_CTL_OP_READ_LIMIT 2 +#define CBQRI_BC_ALLOC_CTL_STATUS_SUCCESS 1 + +int qos_resctrl_setup(void); +int qos_resctrl_online_cpu(unsigned int cpu); +int qos_resctrl_offline_cpu(unsigned int cpu); + +struct cbqri_resctrl_res { + struct rdt_resource resctrl_res; + u32 max_rcid; + u32 max_mcid; +}; + +struct cbqri_resctrl_dom { + struct rdt_ctrl_domain resctrl_ctrl_dom; + struct cbqri_controller *hw_ctrl; +}; + +struct cbqri_config { + u64 cbm; /* capacity block mask */ + u64 rbwb; /* reserved bandwidth blocks */ +}; + +#endif /* _ASM_RISCV_QOS_INTERNAL_H */ diff --git a/arch/riscv/kernel/qos/qos.c b/arch/riscv/kernel/qos/qos.c new file mode 100644 index 00000000000000..560607abf10acd --- /dev/null +++ b/arch/riscv/kernel/qos/qos.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include + +#include +#include + +#include "internal.h" + +/* cached value of srmcfg csr for each cpu */ +DEFINE_PER_CPU(u32, cpu_srmcfg); + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DEFINE_PER_CPU(u32, cpu_srmcfg_default); + +static int __init qos_arch_late_init(void) +{ + int err; + + if (!riscv_isa_extension_available(NULL, SSQOSID)) + return -ENODEV; + + err = qos_resctrl_setup(); + if (err) + return err; + + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "qos:online", + qos_resctrl_online_cpu, + qos_resctrl_offline_cpu); + if (err < 0) { + resctrl_exit(); + return err; + } + + return 0; +} +late_initcall(qos_arch_late_init); diff --git a/arch/riscv/kernel/qos/qos_resctrl.c b/arch/riscv/kernel/qos/qos_resctrl.c new file mode 100644 index 00000000000000..8d7e3b0abb75ae --- /dev/null +++ b/arch/riscv/kernel/qos/qos_resctrl.c @@ -0,0 +1,1092 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define pr_fmt(fmt) "qos: resctrl: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static struct cbqri_resctrl_res cbqri_resctrl_resources[RDT_NUM_RESOURCES]; + +static bool exposed_alloc_capable; +/* CDP (code data prioritization) on x86 is AT (access type) on RISC-V */ +static bool exposed_cdp_l2_capable; +static bool exposed_cdp_l3_capable; +static bool is_cdp_l2_enabled; +static bool is_cdp_l3_enabled; + +/* used by resctrl_arch_system_num_rmid_idx() */ +static u32 max_rmid; + +LIST_HEAD(cbqri_controllers); + +static int cbqri_wait_busy_flag(struct cbqri_controller *ctrl, int reg_offset, + u64 *regp); + +/* Set capacity block mask (cc_block_mask) */ +static void cbqri_set_cbm(struct cbqri_controller *ctrl, u64 cbm) +{ + iowrite64(cbm, ctrl->base + CBQRI_CC_BLOCK_MASK_OFF); +} + +/* Set the Rbwb (reserved bandwidth blocks) field in bc_bw_alloc */ +static void cbqri_set_rbwb(struct cbqri_controller *ctrl, u64 rbwb) +{ + u64 reg; + + reg = ioread64(ctrl->base + CBQRI_BC_BW_ALLOC_OFF); + reg &= ~CBQRI_CONTROL_REGISTERS_RBWB_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_RBWB_MASK, rbwb); + iowrite64(reg, ctrl->base + CBQRI_BC_BW_ALLOC_OFF); +} + +/* Get the Rbwb (reserved bandwidth blocks) field in bc_bw_alloc */ +static u64 cbqri_get_rbwb(struct cbqri_controller *ctrl) +{ + u64 reg; + + reg = ioread64(ctrl->base + CBQRI_BC_BW_ALLOC_OFF); + return FIELD_GET(CBQRI_CONTROL_REGISTERS_RBWB_MASK, reg); +} + +static int cbqri_wait_busy_flag(struct cbqri_controller *ctrl, int reg_offset, + u64 *regp) +{ + u64 reg; + int ret; + + ret = readq_poll_timeout_atomic(ctrl->base + reg_offset, reg, + !FIELD_GET(CBQRI_CONTROL_REGISTERS_BUSY_MASK, reg), + 0, 1000); + if (!ret && regp) + *regp = reg; + + return ret; +} + +/* Perform capacity allocation control operation on capacity controller */ +static int cbqri_cc_alloc_op(struct cbqri_controller *ctrl, int operation, int rcid, + enum resctrl_conf_type type) +{ + int reg_offset = CBQRI_CC_ALLOC_CTL_OFF; + int status; + u64 reg; + + reg = ioread64(ctrl->base + reg_offset); + reg &= ~CBQRI_CONTROL_REGISTERS_OP_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_OP_MASK, operation); + reg &= ~CBQRI_CONTROL_REGISTERS_RCID_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_RCID_MASK, rcid); + + /* CBQRI capacity AT is only supported on L2 and L3 caches for now */ + if (ctrl->type == CBQRI_CONTROLLER_TYPE_CAPACITY && + ((ctrl->cache.cache_level == 2 && is_cdp_l2_enabled) || + (ctrl->cache.cache_level == 3 && is_cdp_l3_enabled))) { + reg &= ~CBQRI_CONTROL_REGISTERS_AT_MASK; + switch (type) { + case CDP_CODE: + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_AT_MASK, + CBQRI_CONTROL_REGISTERS_AT_CODE); + break; + case CDP_DATA: + default: + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_AT_MASK, + CBQRI_CONTROL_REGISTERS_AT_DATA); + break; + } + } + + iowrite64(reg, ctrl->base + reg_offset); + + if (cbqri_wait_busy_flag(ctrl, reg_offset, ®) < 0) { + pr_err("%s(): BUSY timeout when executing the operation\n", __func__); + return -EIO; + } + + status = FIELD_GET(CBQRI_CONTROL_REGISTERS_STATUS_MASK, reg); + if (status != CBQRI_CC_ALLOC_CTL_STATUS_SUCCESS) { + pr_err("%s(): operation %d failed: status=%d\n", __func__, operation, status); + return -EIO; + } + + return 0; +} + +/* + * Write a capacity block mask and verify the hardware accepted it by + * reading back the value after a CONFIG_LIMIT + READ_LIMIT sequence. + */ +static int cbqri_apply_cache_config(struct cbqri_resctrl_dom *hw_dom, u32 closid, + enum resctrl_conf_type type, struct cbqri_config *cfg) +{ + struct cbqri_controller *ctrl = hw_dom->hw_ctrl; + int err = 0; + u64 reg; + + spin_lock(&ctrl->lock); + + /* Set capacity block mask (cc_block_mask) */ + cbqri_set_cbm(ctrl, cfg->cbm); + + /* Capacity config limit operation */ + err = cbqri_cc_alloc_op(ctrl, CBQRI_CC_ALLOC_CTL_OP_CONFIG_LIMIT, closid, type); + if (err < 0) { + pr_err("%s(): operation failed: err = %d\n", __func__, err); + goto out; + } + + /* Clear cc_block_mask before read limit to verify op works */ + cbqri_set_cbm(ctrl, 0); + + /* Perform a capacity read limit operation to verify blockmask */ + err = cbqri_cc_alloc_op(ctrl, CBQRI_CC_ALLOC_CTL_OP_READ_LIMIT, closid, type); + if (err < 0) { + pr_err("%s(): operation failed: err = %d\n", __func__, err); + goto out; + } + + /* Read capacity blockmask to verify it matches the requested config */ + reg = ioread64(ctrl->base + CBQRI_CC_BLOCK_MASK_OFF); + if (reg != cfg->cbm) { + pr_err("%s(): failed to verify allocation (reg:%llx != cbm:%llx)\n", + __func__, reg, cfg->cbm); + err = -EIO; + } + +out: + spin_unlock(&ctrl->lock); + return err; +} + +/* Perform bandwidth allocation control operation on bandwidth controller */ +static int cbqri_bc_alloc_op(struct cbqri_controller *ctrl, int operation, int rcid) +{ + int reg_offset = CBQRI_BC_ALLOC_CTL_OFF; + int status; + u64 reg; + + reg = ioread64(ctrl->base + reg_offset); + reg &= ~CBQRI_CONTROL_REGISTERS_OP_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_OP_MASK, operation); + reg &= ~CBQRI_CONTROL_REGISTERS_RCID_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_RCID_MASK, rcid); + iowrite64(reg, ctrl->base + reg_offset); + + if (cbqri_wait_busy_flag(ctrl, reg_offset, ®) < 0) { + pr_err("%s(): BUSY timeout when executing the operation\n", __func__); + return -EIO; + } + + status = FIELD_GET(CBQRI_CONTROL_REGISTERS_STATUS_MASK, reg); + if (status != CBQRI_BC_ALLOC_CTL_STATUS_SUCCESS) { + pr_err("%s(): operation %d failed with status = %d\n", + __func__, operation, status); + return -EIO; + } + + return 0; +} + +/* + * Write a bandwidth reservation and verify the hardware accepted it by + * reading back the value after a CONFIG_LIMIT + READ_LIMIT sequence. + */ +static int cbqri_apply_bw_config(struct cbqri_resctrl_dom *hw_dom, u32 closid, + enum resctrl_conf_type type, struct cbqri_config *cfg) +{ + struct cbqri_controller *ctrl = hw_dom->hw_ctrl; + int ret = 0; + u64 reg; + + spin_lock(&ctrl->lock); + + /* Set reserved bandwidth blocks */ + cbqri_set_rbwb(ctrl, cfg->rbwb); + + /* Bandwidth config limit operation */ + ret = cbqri_bc_alloc_op(ctrl, CBQRI_BC_ALLOC_CTL_OP_CONFIG_LIMIT, closid); + if (ret < 0) { + pr_err("%s(): operation failed: ret = %d\n", __func__, ret); + goto out; + } + + /* Clear rbwb before read limit to verify op works */ + cbqri_set_rbwb(ctrl, 0); + + /* Bandwidth allocation read limit operation to verify */ + ret = cbqri_bc_alloc_op(ctrl, CBQRI_BC_ALLOC_CTL_OP_READ_LIMIT, closid); + if (ret < 0) + goto out; + + /* Read bandwidth allocation to verify it matches the requested config */ + reg = cbqri_get_rbwb(ctrl); + if (reg != cfg->rbwb) { + pr_err("%s(): failed to verify allocation (reg:%llx != rbwb:%llu)\n", + __func__, reg, cfg->rbwb); + ret = -EIO; + } + +out: + spin_unlock(&ctrl->lock); + return ret; +} + +static int cbqri_probe_feature(struct cbqri_controller *ctrl, int reg_offset, + int operation, int *status, bool *access_type_supported) +{ + u64 reg, saved_reg; + int at; + + /* Keep the initial register value to preserve the WPRI fields */ + reg = ioread64(ctrl->base + reg_offset); + saved_reg = reg; + + /* Execute the requested operation to find if the register is implemented */ + reg &= ~CBQRI_CONTROL_REGISTERS_OP_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_OP_MASK, operation); + iowrite64(reg, ctrl->base + reg_offset); + if (cbqri_wait_busy_flag(ctrl, reg_offset, ®) < 0) { + pr_err("%s(): BUSY timeout when executing the operation\n", __func__); + return -EIO; + } + + /* Get the operation status */ + *status = FIELD_GET(CBQRI_CONTROL_REGISTERS_STATUS_MASK, reg); + + /* + * Check for the AT support if the register is implemented + * (if not, the status value will remain 0) + */ + if (*status != 0) { + /* Set the AT field to a valid value */ + reg = saved_reg; + reg &= ~CBQRI_CONTROL_REGISTERS_AT_MASK; + reg |= FIELD_PREP(CBQRI_CONTROL_REGISTERS_AT_MASK, + CBQRI_CONTROL_REGISTERS_AT_CODE); + iowrite64(reg, ctrl->base + reg_offset); + if (cbqri_wait_busy_flag(ctrl, reg_offset, ®) < 0) { + pr_err("%s(): BUSY timeout when setting AT field\n", __func__); + return -EIO; + } + + /* + * If the AT field value has been reset to zero, + * then the AT support is not present + */ + at = FIELD_GET(CBQRI_CONTROL_REGISTERS_AT_MASK, reg); + if (at == CBQRI_CONTROL_REGISTERS_AT_CODE) + *access_type_supported = true; + else + *access_type_supported = false; + } + + /* Restore the original register value */ + iowrite64(saved_reg, ctrl->base + reg_offset); + if (cbqri_wait_busy_flag(ctrl, reg_offset, NULL) < 0) { + pr_err("%s(): BUSY timeout when restoring the original register value\n", __func__); + return -EIO; + } + + return 0; +} + +static int cbqri_probe_cc(struct cbqri_controller *ctrl) +{ + int err, status; + u64 reg; + + reg = ioread64(ctrl->base + CBQRI_CC_CAPABILITIES_OFF); + if (reg == 0) + return -ENODEV; + + ctrl->ver_minor = FIELD_GET(CBQRI_CC_CAPABILITIES_VER_MINOR_MASK, reg); + ctrl->ver_major = FIELD_GET(CBQRI_CC_CAPABILITIES_VER_MAJOR_MASK, reg); + ctrl->cc.supports_alloc_op_flush_rcid = + FIELD_GET(CBQRI_CC_CAPABILITIES_FRCID_MASK, reg); + ctrl->cc.ncblks = FIELD_GET(CBQRI_CC_CAPABILITIES_NCBLKS_MASK, reg); + + pr_debug("version=%d.%d ncblks=%d cache_level=%d\n", + ctrl->ver_major, ctrl->ver_minor, + ctrl->cc.ncblks, ctrl->cache.cache_level); + + /* Probe allocation features (monitoring not yet implemented) */ + err = cbqri_probe_feature(ctrl, CBQRI_CC_ALLOC_CTL_OFF, + CBQRI_CC_ALLOC_CTL_OP_READ_LIMIT, + &status, &ctrl->cc.supports_alloc_at_code); + if (err) + return err; + + if (status == CBQRI_CC_ALLOC_CTL_STATUS_SUCCESS) { + ctrl->alloc_capable = true; + exposed_alloc_capable = true; + } + + return 0; +} + +static int cbqri_probe_bc(struct cbqri_controller *ctrl) +{ + int err, status; + u64 reg; + + reg = ioread64(ctrl->base + CBQRI_BC_CAPABILITIES_OFF); + if (reg == 0) + return -ENODEV; + + ctrl->ver_minor = FIELD_GET(CBQRI_BC_CAPABILITIES_VER_MINOR_MASK, reg); + ctrl->ver_major = FIELD_GET(CBQRI_BC_CAPABILITIES_VER_MAJOR_MASK, reg); + ctrl->bc.nbwblks = FIELD_GET(CBQRI_BC_CAPABILITIES_NBWBLKS_MASK, reg); + ctrl->bc.mrbwb = FIELD_GET(CBQRI_BC_CAPABILITIES_MRBWB_MASK, reg); + + if (!ctrl->bc.nbwblks) { + pr_err("bandwidth controller has nbwblks=0\n"); + return -EINVAL; + } + + pr_debug("version=%d.%d nbwblks=%d mrbwb=%d\n", + ctrl->ver_major, ctrl->ver_minor, + ctrl->bc.nbwblks, ctrl->bc.mrbwb); + + /* Probe allocation features (monitoring not yet implemented) */ + err = cbqri_probe_feature(ctrl, CBQRI_BC_ALLOC_CTL_OFF, + CBQRI_BC_ALLOC_CTL_OP_READ_LIMIT, + &status, &ctrl->bc.supports_alloc_at_code); + if (err) + return err; + + if (status == CBQRI_BC_ALLOC_CTL_STATUS_SUCCESS) { + ctrl->alloc_capable = true; + exposed_alloc_capable = true; + } + + return 0; +} + +static int cbqri_probe_controller(struct cbqri_controller *ctrl) +{ + int err; + + pr_debug("controller info: type=%d addr=%pa size=%pa max-rcid=%u max-mcid=%u\n", + ctrl->type, &ctrl->addr, &ctrl->size, + ctrl->rcid_count, ctrl->mcid_count); + + if (!ctrl->addr) { + pr_warn("%s(): controller has invalid addr=0x0, skipping\n", __func__); + return -EINVAL; + } + + if (!request_mem_region(ctrl->addr, ctrl->size, "cbqri_controller")) { + pr_err("%s(): request_mem_region failed for %pa\n", + __func__, &ctrl->addr); + return -EBUSY; + } + + ctrl->base = ioremap(ctrl->addr, ctrl->size); + if (!ctrl->base) { + pr_err("%s(): ioremap failed for %pa\n", __func__, &ctrl->addr); + err = -ENOMEM; + goto err_release; + } + + spin_lock_init(&ctrl->lock); + + switch (ctrl->type) { + case CBQRI_CONTROLLER_TYPE_CAPACITY: + err = cbqri_probe_cc(ctrl); + break; + case CBQRI_CONTROLLER_TYPE_BANDWIDTH: + err = cbqri_probe_bc(ctrl); + break; + default: + pr_err("unknown controller type %d\n", ctrl->type); + err = -ENODEV; + break; + } + + if (err) + goto err_iounmap; + + /* + * max_rmid is used by resctrl_arch_system_num_rmid_idx() + * Find the smallest mcid_count amongst all controllers. + */ + max_rmid = min(max_rmid, ctrl->mcid_count); + + return 0; + +err_iounmap: + iounmap(ctrl->base); + ctrl->base = NULL; +err_release: + release_mem_region(ctrl->addr, ctrl->size); + return err; +} + +bool resctrl_arch_alloc_capable(void) +{ + return exposed_alloc_capable; +} + +bool resctrl_arch_mon_capable(void) +{ + /* Monitoring not yet implemented */ + return false; +} + +bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) +{ + switch (rid) { + case RDT_RESOURCE_L2: + return is_cdp_l2_enabled; + + case RDT_RESOURCE_L3: + return is_cdp_l3_enabled; + + default: + return false; + } +} + +int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) +{ + switch (rid) { + case RDT_RESOURCE_L2: + if (!exposed_cdp_l2_capable) + return -ENODEV; + is_cdp_l2_enabled = enable; + break; + + case RDT_RESOURCE_L3: + if (!exposed_cdp_l3_capable) + return -ENODEV; + is_cdp_l3_enabled = enable; + break; + + default: + return -ENODEV; + } + + return 0; +} + +struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) +{ + if (l >= RDT_NUM_RESOURCES) + return NULL; + + return &cbqri_resctrl_resources[l].resctrl_res; +} + +bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) +{ + return false; +} + +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, + enum resctrl_event_id evtid) +{ + /* RISC-V can always read an rmid, nothing needs allocating */ + return NULL; +} + +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, + enum resctrl_event_id evtid, void *arch_mon_ctx) +{ + /* No arch-private monitoring context to free */ +} + +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ + /* MBM counter assignment not supported */ +} + +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 unused, u32 rmid, int cntr_id, + enum resctrl_event_id eventid, u64 *val) +{ + /* MBM counter assignment not supported */ + return -EOPNOTSUPP; +} + +bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) +{ + /* MBM counter assignment not supported */ + return false; +} + +int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) +{ + /* MBM counter assignment not supported */ + return 0; +} + +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 unused, u32 rmid, int cntr_id, + enum resctrl_event_id eventid) +{ + /* MBM counter assignment not supported */ +} + +bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r) +{ + /* CBQRI does not have I/O-specific allocation */ + return false; +} + +int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable) +{ + /* CBQRI does not have I/O-specific allocation */ + return 0; +} + +/* + * Note about terminology between x86 (Intel RDT/AMD QoS) and RISC-V: + * CLOSID on x86 is RCID on RISC-V + * RMID on x86 is MCID on RISC-V + */ +u32 resctrl_arch_get_num_closid(struct rdt_resource *res) +{ + struct cbqri_resctrl_res *hw_res; + + hw_res = container_of(res, struct cbqri_resctrl_res, resctrl_res); + + return hw_res->max_rcid; +} + +u32 resctrl_arch_system_num_rmid_idx(void) +{ + return max_rmid; +} + +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid) +{ + return rmid; +} + +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid) +{ + *closid = RISCV_RESCTRL_EMPTY_CLOSID; + *rmid = idx; +} + +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid) +{ + u32 srmcfg; + + WARN_ON_ONCE((closid & SRMCFG_RCID_MASK) != closid); + WARN_ON_ONCE((rmid & SRMCFG_MCID_MASK) != rmid); + + srmcfg = rmid << SRMCFG_MCID_SHIFT; + srmcfg |= closid; + WRITE_ONCE(per_cpu(cpu_srmcfg_default, cpu), srmcfg); +} + +void resctrl_arch_sched_in(struct task_struct *tsk) +{ + __switch_to_srmcfg(tsk); +} + +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) +{ + u32 srmcfg; + + WARN_ON_ONCE((closid & SRMCFG_RCID_MASK) != closid); + WARN_ON_ONCE((rmid & SRMCFG_MCID_MASK) != rmid); + + srmcfg = rmid << SRMCFG_MCID_SHIFT; + srmcfg |= closid; + WRITE_ONCE(tsk->thread.srmcfg, srmcfg); +} + +void resctrl_arch_sync_cpu_closid_rmid(void *info) +{ + struct resctrl_cpu_defaults *r = info; + + lockdep_assert_preemption_disabled(); + + if (r) { + resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), + r->closid, r->rmid); + } + + resctrl_arch_sched_in(current); +} + +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid) +{ + return (READ_ONCE(tsk->thread.srmcfg) & SRMCFG_RCID_MASK) == closid; +} + +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid) +{ + u32 tsk_rmid; + + tsk_rmid = READ_ONCE(tsk->thread.srmcfg); + tsk_rmid >>= SRMCFG_MCID_SHIFT; + tsk_rmid &= SRMCFG_MCID_MASK; + + return tsk_rmid == rmid; +} + +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, + u32 closid, u32 rmid, enum resctrl_event_id eventid, + void *arch_priv, u64 *val, void *arch_mon_ctx) +{ + /* + * Cache occupancy and bandwidth monitoring are not yet implemented + * for RISC-V CBQRI. This will be added in a future series once the + * resctrl framework supports monitoring domains at non-L3 scopes. + */ + return -EOPNOTSUPP; +} + +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, enum resctrl_event_id eventid) +{ + /* Monitoring not yet supported; nothing to reset */ +} + +void resctrl_arch_mon_event_config_read(void *info) +{ + /* Monitoring not yet supported; no event config */ +} + +void resctrl_arch_mon_event_config_write(void *info) +{ + /* Monitoring not yet supported; no event config */ +} + +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) +{ + /* Monitoring not yet supported; nothing to reset */ +} + +void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) +{ + struct cbqri_resctrl_res *hw_res; + struct rdt_ctrl_domain *d; + enum resctrl_conf_type t; + u32 default_ctrl; + int i; + + lockdep_assert_cpus_held(); + + hw_res = container_of(r, struct cbqri_resctrl_res, resctrl_res); + default_ctrl = resctrl_get_default_ctrl(r); + + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + for (i = 0; i < hw_res->max_rcid; i++) { + for (t = 0; t < CDP_NUM_TYPES; t++) + resctrl_arch_update_one(r, d, i, t, default_ctrl); + } + } +} + +void resctrl_arch_pre_mount(void) +{ + /* All controllers discovered at boot via late_initcall; nothing to do */ +} + +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, + u32 closid, enum resctrl_conf_type t, u32 cfg_val) +{ + struct cbqri_controller *ctrl; + struct cbqri_resctrl_dom *dom; + struct cbqri_config cfg; + int err = 0; + + dom = container_of(d, struct cbqri_resctrl_dom, resctrl_ctrl_dom); + ctrl = dom->hw_ctrl; + + if (!r->alloc_capable) + return -EINVAL; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + cfg.cbm = cfg_val; + err = cbqri_apply_cache_config(dom, closid, t, &cfg); + break; + case RDT_RESOURCE_MBA: + /* convert from percentage to bandwidth blocks */ + cfg.rbwb = cfg_val * ctrl->bc.nbwblks / 100; + cfg.rbwb = min_t(u64, cfg.rbwb, ctrl->bc.mrbwb); + err = cbqri_apply_bw_config(dom, closid, t, &cfg); + break; + default: + return -EINVAL; + } + + return err; +} + +int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) +{ + struct resctrl_staged_config *cfg; + enum resctrl_conf_type t; + struct rdt_ctrl_domain *d; + int err = 0; + + /* Walking r->ctrl_domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + for (t = 0; t < CDP_NUM_TYPES; t++) { + cfg = &d->staged_config[t]; + if (!cfg->have_new_ctrl) + continue; + err = resctrl_arch_update_one(r, d, closid, t, cfg->new_ctrl); + if (err) { + pr_err("%s(): update failed (err=%d)\n", __func__, err); + return err; + } + } + } + return err; +} + +u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, + u32 closid, enum resctrl_conf_type type) +{ + struct cbqri_resctrl_dom *hw_dom; + struct cbqri_controller *ctrl; + u32 val; + u32 rbwb; + int err; + + hw_dom = container_of(d, struct cbqri_resctrl_dom, resctrl_ctrl_dom); + + ctrl = hw_dom->hw_ctrl; + + val = resctrl_get_default_ctrl(r); + + if (!r->alloc_capable) + return val; + + spin_lock(&ctrl->lock); + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + /* Clear cc_block_mask before read limit operation */ + cbqri_set_cbm(ctrl, 0); + + /* Capacity read limit operation for RCID (closid) */ + err = cbqri_cc_alloc_op(ctrl, CBQRI_CC_ALLOC_CTL_OP_READ_LIMIT, closid, type); + if (err < 0) { + pr_err("%s(): operation failed: err = %d\n", __func__, err); + break; + } + + /* Read capacity block mask for RCID (closid) */ + val = ioread64(ctrl->base + CBQRI_CC_BLOCK_MASK_OFF); + break; + + case RDT_RESOURCE_MBA: + /* Bandwidth read limit operation for RCID (closid) */ + err = cbqri_bc_alloc_op(ctrl, CBQRI_BC_ALLOC_CTL_OP_READ_LIMIT, closid); + if (err < 0) { + pr_err("%s(): operation failed: err = %d\n", __func__, err); + break; + } + + rbwb = cbqri_get_rbwb(ctrl); + val = DIV_ROUND_UP(rbwb * 100, ctrl->bc.nbwblks); + break; + + default: + break; + } + + spin_unlock(&ctrl->lock); + return val; +} + +static struct rdt_ctrl_domain *qos_new_domain(struct cbqri_controller *ctrl) +{ + struct cbqri_resctrl_dom *hw_dom; + struct rdt_ctrl_domain *domain; + + hw_dom = kzalloc_obj(*hw_dom, GFP_KERNEL); + if (!hw_dom) + return NULL; + + /* associate this cbqri_controller with the domain */ + hw_dom->hw_ctrl = ctrl; + + /* the rdt_domain struct from inside the cbqri_resctrl_dom struct */ + domain = &hw_dom->resctrl_ctrl_dom; + + INIT_LIST_HEAD(&domain->hdr.list); + + return domain; +} + +static int qos_init_domain_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain *d) +{ + struct cbqri_resctrl_res *hw_res; + int err = 0; + int i; + + hw_res = container_of(r, struct cbqri_resctrl_res, resctrl_res); + + for (i = 0; i < hw_res->max_rcid; i++) { + err = resctrl_arch_update_one(r, d, i, 0, resctrl_get_default_ctrl(r)); + if (err) + return err; + } + return 0; +} + +static int qos_init_cache_resource(struct cbqri_controller *ctrl, + struct cbqri_resctrl_res *cbqri_res, + enum resctrl_res_level rid, char *name, + enum resctrl_scope scope) +{ + struct rdt_resource *res = &cbqri_res->resctrl_res; + + /* Already initialized by a previous controller at this cache level */ + if (res->name) { + if (cbqri_res->max_rcid != ctrl->rcid_count || + res->cache.cbm_len != ctrl->cc.ncblks) { + pr_err("%s controllers have mismatched capabilities\n", + name); + return -EINVAL; + } + return 0; + } + + cbqri_res->max_rcid = ctrl->rcid_count; + cbqri_res->max_mcid = ctrl->mcid_count; + res->rid = rid; + res->name = name; + res->alloc_capable = ctrl->alloc_capable; + res->schema_fmt = RESCTRL_SCHEMA_BITMAP; + res->ctrl_scope = scope; + res->cache.cbm_len = ctrl->cc.ncblks; + res->cache.shareable_bits = resctrl_get_default_ctrl(res); + res->cache.min_cbm_bits = 1; + return 0; +} + +static int qos_init_membw_resource(struct cbqri_controller *ctrl, + struct cbqri_resctrl_res *cbqri_res) +{ + struct rdt_resource *res = &cbqri_res->resctrl_res; + + if (res->name) { + if (cbqri_res->max_rcid != ctrl->rcid_count || + res->membw.max_bw != DIV_ROUND_UP(ctrl->bc.mrbwb * 100, + ctrl->bc.nbwblks)) { + pr_err("MB controllers have mismatched capabilities\n"); + return -EINVAL; + } + return 0; + } + + cbqri_res->max_rcid = ctrl->rcid_count; + cbqri_res->max_mcid = ctrl->mcid_count; + res->rid = RDT_RESOURCE_MBA; + res->name = "MB"; + res->alloc_capable = ctrl->alloc_capable; + res->schema_fmt = RESCTRL_SCHEMA_RANGE; + /* + * resctrl requires a cache scope for MBA domains. Use L3 as a + * proxy until the framework supports non-cache scopes for + * bandwidth resources. + */ + res->ctrl_scope = RESCTRL_L3_CACHE; + res->membw.delay_linear = true; + res->membw.arch_needs_linear = true; + res->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; + res->membw.min_bw = 1; + res->membw.max_bw = DIV_ROUND_UP(ctrl->bc.mrbwb * 100, ctrl->bc.nbwblks); + res->membw.bw_gran = 1; + return 0; +} + +static int qos_resctrl_add_controller_domain(struct cbqri_controller *ctrl) +{ + struct rdt_ctrl_domain *domain; + struct cbqri_resctrl_res *cbqri_res = NULL; + struct rdt_resource *res = NULL; + struct list_head *pos = NULL; + int err; + + domain = qos_new_domain(ctrl); + if (!domain) + return -ENOSPC; + + switch (ctrl->type) { + case CBQRI_CONTROLLER_TYPE_CAPACITY: + cpumask_copy(&domain->hdr.cpu_mask, &ctrl->cache.cpu_mask); + domain->hdr.id = ctrl->cache.cache_id; + + if (ctrl->cache.cache_level == 2) { + cbqri_res = &cbqri_resctrl_resources[RDT_RESOURCE_L2]; + err = qos_init_cache_resource(ctrl, cbqri_res, + RDT_RESOURCE_L2, "L2", + RESCTRL_L2_CACHE); + } else if (ctrl->cache.cache_level == 3) { + cbqri_res = &cbqri_resctrl_resources[RDT_RESOURCE_L3]; + err = qos_init_cache_resource(ctrl, cbqri_res, + RDT_RESOURCE_L3, "L3", + RESCTRL_L3_CACHE); + } else { + pr_err("unknown cache level %d\n", ctrl->cache.cache_level); + err = -ENODEV; + } + if (err) + goto err_free_domain; + res = &cbqri_res->resctrl_res; + break; + + case CBQRI_CONTROLLER_TYPE_BANDWIDTH: + cpumask_copy(&domain->hdr.cpu_mask, &ctrl->mem.cpu_mask); + domain->hdr.id = ctrl->mem.prox_dom; + if (ctrl->alloc_capable) { + cbqri_res = &cbqri_resctrl_resources[RDT_RESOURCE_MBA]; + err = qos_init_membw_resource(ctrl, cbqri_res); + if (err) + goto err_free_domain; + res = &cbqri_res->resctrl_res; + } + break; + + default: + pr_err("unknown controller type %d\n", ctrl->type); + err = -ENODEV; + goto err_free_domain; + } + + if (!res) + goto out; + + err = qos_init_domain_ctrlval(res, domain); + if (err) + goto err_free_domain; + + if (resctrl_find_domain(&res->ctrl_domains, domain->hdr.id, &pos)) { + pr_err("duplicate domain id %d for resource %s\n", + domain->hdr.id, res->name); + err = -EEXIST; + goto err_free_domain; + } + if (pos) + list_add_tail(&domain->hdr.list, pos); + else + list_add_tail(&domain->hdr.list, &res->ctrl_domains); + + err = resctrl_online_ctrl_domain(res, domain); + if (err) { + pr_err("failed to online domain %d\n", domain->hdr.id); + list_del(&domain->hdr.list); + goto err_free_domain; + } + +out: + return 0; + +err_free_domain: + kfree(container_of(domain, struct cbqri_resctrl_dom, resctrl_ctrl_dom)); + return err; +} + +int qos_resctrl_setup(void) +{ + struct rdt_ctrl_domain *domain, *domain_temp; + struct cbqri_controller *ctrl; + struct cbqri_resctrl_res *res; + int err = 0; + int i = 0; + + max_rmid = U32_MAX; + + for (i = 0; i < RDT_NUM_RESOURCES; i++) { + res = &cbqri_resctrl_resources[i]; + INIT_LIST_HEAD(&res->resctrl_res.ctrl_domains); + INIT_LIST_HEAD(&res->resctrl_res.mon_domains); + res->resctrl_res.rid = i; + } + + list_for_each_entry(ctrl, &cbqri_controllers, list) { + err = cbqri_probe_controller(ctrl); + if (err) { + pr_err("%s(): failed (%d)\n", __func__, err); + goto err_free_controllers_list; + } + + err = qos_resctrl_add_controller_domain(ctrl); + if (err) { + pr_err("%s(): failed to add controller domain (%d)\n", __func__, err); + goto err_free_controllers_list; + } + + /* + * CDP (code data prioritization) on x86 is similar to + * the AT (access type) field in CBQRI. CDP only supports + * caches so this must be a CBQRI capacity controller. + */ + if (ctrl->type == CBQRI_CONTROLLER_TYPE_CAPACITY && + ctrl->cc.supports_alloc_at_code) { + if (ctrl->cache.cache_level == 2) + exposed_cdp_l2_capable = true; + else + exposed_cdp_l3_capable = true; + } + } + pr_debug("alloc=%d cdp_l2=%d cdp_l3=%d\n", + exposed_alloc_capable, + exposed_cdp_l2_capable, exposed_cdp_l3_capable); + + err = resctrl_init(); + if (err) + goto err_free_controllers_list; + + return 0; + +err_free_controllers_list: + for (i = 0; i < RDT_NUM_RESOURCES; i++) { + res = &cbqri_resctrl_resources[i]; + list_for_each_entry_safe(domain, domain_temp, &res->resctrl_res.ctrl_domains, + hdr.list) { + resctrl_offline_ctrl_domain(&res->resctrl_res, domain); + list_del(&domain->hdr.list); + kfree(container_of(domain, struct cbqri_resctrl_dom, resctrl_ctrl_dom)); + } + } + + list_for_each_entry(ctrl, &cbqri_controllers, list) { + if (!ctrl->base) + break; + iounmap(ctrl->base); + ctrl->base = NULL; + release_mem_region(ctrl->addr, ctrl->size); + } + + return err; +} + +int qos_resctrl_online_cpu(unsigned int cpu) +{ + resctrl_online_cpu(cpu); + return 0; +} + +int qos_resctrl_offline_cpu(unsigned int cpu) +{ + resctrl_offline_cpu(cpu); + return 0; +} diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c index de5f8c018333d7..36e375551b43db 100644 --- a/drivers/acpi/pptt.c +++ b/drivers/acpi/pptt.c @@ -1063,3 +1063,66 @@ int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus) return 0; } + +/** + * acpi_pptt_get_cache_size_from_id() - Get the size of the specified cache + * @cache_id: The id field of the cache + * @size: Where to store the cache size in bytes + * + * Determine the size of the cache identified by cache_id. This allows the + * property to be found even if the CPUs are offline. + * + * The PPTT table must be rev 3 or later. + * + * Return: -ENOENT if the PPTT doesn't exist, the revision isn't supported or + * the cache cannot be found. Otherwise returns 0 and sets *size. + */ +int acpi_pptt_get_cache_size_from_id(u32 cache_id, u32 *size) +{ + int cpu; + struct acpi_table_header *table; + + table = acpi_get_pptt(); + if (!table) + return -ENOENT; + + if (table->revision < 3) + return -ENOENT; + + for_each_possible_cpu(cpu) { + bool empty; + int level = 1; + u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu); + struct acpi_pptt_processor *cpu_node; + + cpu_node = acpi_find_processor_node(table, acpi_cpu_id); + if (!cpu_node) + continue; + + do { + int cache_type[] = {CACHE_TYPE_INST, CACHE_TYPE_DATA, CACHE_TYPE_UNIFIED}; + + empty = true; + for (int i = 0; i < ARRAY_SIZE(cache_type); i++) { + struct acpi_pptt_cache *cache; + struct acpi_pptt_cache_v1_full *cache_v1; + + cache = acpi_find_cache_node(table, acpi_cpu_id, cache_type[i], + level, &cpu_node); + if (!cache) + continue; + + empty = false; + + cache_v1 = upgrade_pptt_cache(cache); + if (cache_v1 && cache_v1->cache_id == cache_id) { + *size = cache->size; + return 0; + } + } + level++; + } while (!empty); + } + + return -ENOENT; +} diff --git a/drivers/acpi/riscv/Makefile b/drivers/acpi/riscv/Makefile index 1284a076fa8887..d7ae8729987a3a 100644 --- a/drivers/acpi/riscv/Makefile +++ b/drivers/acpi/riscv/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y += rhct.o init.o irq.o +obj-$(CONFIG_RISCV_ISA_SSQOSID) += rqsc.o obj-$(CONFIG_ACPI_PROCESSOR_IDLE) += cpuidle.o obj-$(CONFIG_ACPI_CPPC_LIB) += cppc.o obj-$(CONFIG_ACPI_RIMT) += rimt.o diff --git a/drivers/acpi/riscv/init.c b/drivers/acpi/riscv/init.c index 7c00f7995e866d..8a74dff42dce25 100644 --- a/drivers/acpi/riscv/init.c +++ b/drivers/acpi/riscv/init.c @@ -4,12 +4,35 @@ * Author: Sunil V L */ +#define pr_fmt(fmt) "ACPI: RQSC: " fmt + #include #include "init.h" void __init acpi_arch_init(void) { + struct acpi_table_header *rqsc; + acpi_status status; + int rc; + riscv_acpi_init_gsi_mapping(); + if (IS_ENABLED(CONFIG_ACPI_RIMT)) riscv_acpi_rimt_init(); + + if (IS_ENABLED(CONFIG_RISCV_ISA_SSQOSID) && !acpi_disabled) { + status = acpi_get_table(ACPI_SIG_RQSC, 0, &rqsc); + if (status == AE_NOT_FOUND) { + /* RQSC is optional; silence on systems without it */ + } else if (ACPI_FAILURE(status)) { + pr_err("failed to get ACPI RQSC table: %s\n", + acpi_format_exception(status)); + } else { + rc = acpi_parse_rqsc(rqsc); + if (rc < 0) + pr_err("failed to parse ACPI RQSC table: %d\n", + rc); + acpi_put_table(rqsc); + } + } } diff --git a/drivers/acpi/riscv/rqsc.c b/drivers/acpi/riscv/rqsc.c new file mode 100644 index 00000000000000..f647051be0bf59 --- /dev/null +++ b/drivers/acpi/riscv/rqsc.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Tenstorrent + * Author: Drew Fustini + */ + +#define pr_fmt(fmt) "ACPI: RQSC: " fmt + +#include +#include +#include + +#define CBQRI_CTRL_SIZE 0x1000 + +int __init acpi_parse_rqsc(struct acpi_table_header *table) +{ + struct acpi_table_rqsc *rqsc; + struct acpi_table_rqsc_fields *end; + struct acpi_table_rqsc_fields *node; + int err; + int num_controllers = 0; + + rqsc = (struct acpi_table_rqsc *)table; + + end = ACPI_ADD_PTR(struct acpi_table_rqsc_fields, rqsc, rqsc->header.length); + + for (node = ACPI_ADD_PTR(struct acpi_table_rqsc_fields, rqsc, + sizeof(struct acpi_table_rqsc)); + node < end; + node = ACPI_ADD_PTR(struct acpi_table_rqsc_fields, node, node->length) + ) { + struct cbqri_controller *ctrl; + + if (node->length < sizeof(*node)) { + pr_err("malformed RQSC entry: length %u < %zu, aborting\n", + node->length, sizeof(*node)); + err = -EINVAL; + goto err_free_controllers; + } + + ctrl = kzalloc_obj(*ctrl, GFP_KERNEL); + if (!ctrl) { + err = -ENOMEM; + goto err_free_controllers; + } + + ctrl->type = node->type; + /* reg[1] is the MMIO base address per the RQSC table layout */ + ctrl->addr = node->reg[1]; + ctrl->size = CBQRI_CTRL_SIZE; + ctrl->rcid_count = node->rcid; + ctrl->mcid_count = node->mcid; + + if (!ctrl->addr) { + pr_warn("skipping controller with invalid addr=0x0\n"); + kfree(ctrl); + continue; + } + + if (node->nres == 0) { + pr_warn("controller at %pa has no resource descriptors, skipping\n", + &ctrl->addr); + kfree(ctrl); + continue; + } + + if (node->length < sizeof(*node) + sizeof(node->res[0])) { + pr_warn("controller at %pa: node too short for resource descriptor, skipping\n", + &ctrl->addr); + kfree(ctrl); + continue; + } + + if (node->nres > 1) + pr_warn("controller at %pa has %u resource descriptors, using first\n", + &ctrl->addr, node->nres); + + pr_debug("Found controller with type %u addr %pa size %pa rcid %u mcid %u\n", + ctrl->type, &ctrl->addr, &ctrl->size, + ctrl->rcid_count, ctrl->mcid_count); + if (ctrl->type == CBQRI_CONTROLLER_TYPE_CAPACITY) { + ctrl->cache.cache_id = (u32)node->res[0].id1; + ctrl->cache.cache_level = + find_acpi_cache_level_from_id(ctrl->cache.cache_id); + + if (acpi_pptt_get_cache_size_from_id(ctrl->cache.cache_id, + &ctrl->cache.cache_size)) { + pr_warn("failed to determine size for cache id 0x%x\n", + ctrl->cache.cache_id); + ctrl->cache.cache_size = 0; + } + + pr_debug("Cache controller has ID 0x%x level %u size %u\n", + ctrl->cache.cache_id, ctrl->cache.cache_level, + ctrl->cache.cache_size); + + /* + * For CBQRI, any cpu (technically a hart in RISC-V terms) + * can access the memory-mapped registers of any CBQRI + * controller in the system. + */ + err = acpi_pptt_get_cpumask_from_cache_id(ctrl->cache.cache_id, + &ctrl->cache.cpu_mask); + if (err) { + pr_warn("Failed to get cpumask for cache id 0x%x (%d), skipping\n", + ctrl->cache.cache_id, err); + kfree(ctrl); + continue; + } + + } else if (ctrl->type == CBQRI_CONTROLLER_TYPE_BANDWIDTH) { + ctrl->mem.prox_dom = (u32)node->res[0].id1; + cpumask_copy(&ctrl->mem.cpu_mask, + cpumask_of_node(pxm_to_node(ctrl->mem.prox_dom))); + pr_debug("Memory controller with proximity domain %u\n", + ctrl->mem.prox_dom); + } + + /* List shared with RISC-V QoS resctrl implementation */ + list_add_tail(&ctrl->list, &cbqri_controllers); + num_controllers++; + } + + pr_info("found %d CBQRI controllers\n", num_controllers); + return 0; + +err_free_controllers: + while (!list_empty(&cbqri_controllers)) { + struct cbqri_controller *ctrl; + + ctrl = list_first_entry(&cbqri_controllers, struct cbqri_controller, list); + list_del(&ctrl->list); + kfree(ctrl); + } + return err; +} diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 5c0b55e7b3e4f0..d6f272e848fae7 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -55,6 +55,7 @@ #define ACPI_SIG_RGRT "RGRT" /* Regulatory Graphics Resource Table */ #define ACPI_SIG_RHCT "RHCT" /* RISC-V Hart Capabilities Table */ #define ACPI_SIG_RIMT "RIMT" /* RISC-V IO Mapping Table */ +#define ACPI_SIG_RQSC "RQSC" /* RISC-V Quality of Service Controller */ #define ACPI_SIG_SBST "SBST" /* Smart Battery Specification Table */ #define ACPI_SIG_SDEI "SDEI" /* Software Delegated Exception Interface Table */ #define ACPI_SIG_SDEV "SDEV" /* Secure Devices table */ @@ -3351,6 +3352,41 @@ enum acpi_rgrt_image_type { ACPI_RGRT_TYPE_RESERVED = 2 /* 2 and greater are reserved */ }; +/******************************************************************************* + * + * RQSC - RISC-V Quality of Service Controller + * Version 1 + * + ******************************************************************************/ + +struct acpi_table_rqsc_fields_res { + u8 type; + u8 resv; + u16 length; + u16 flags; + u8 resv2; + u8 id_type; + u64 id1; + u32 id2; +}; + +struct acpi_table_rqsc_fields { + u8 type; + u8 resv; + u16 length; + u32 reg[3]; + u16 rcid; + u16 mcid; + u16 flags; + u16 nres; + struct acpi_table_rqsc_fields_res res[]; +}; + +struct acpi_table_rqsc { + struct acpi_table_header header; /* Common ACPI table header */ + u32 num; +}; + /******************************************************************************* * * RHCT - RISC-V Hart Capabilities Table diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 4d2f0bed7a06da..0596ec18f5227b 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1547,6 +1547,7 @@ int find_acpi_cpu_topology_package(unsigned int cpu); int find_acpi_cpu_topology_hetero_id(unsigned int cpu); void acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, cpumask_t *cpus); int find_acpi_cache_level_from_id(u32 cache_id); +int acpi_pptt_get_cache_size_from_id(u32 cache_id, u32 *size); int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus); #else static inline int acpi_pptt_cpu_is_thread(unsigned int cpu) @@ -1571,10 +1572,17 @@ static inline int find_acpi_cpu_topology_hetero_id(unsigned int cpu) } static inline void acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, cpumask_t *cpus) { } + static inline int find_acpi_cache_level_from_id(u32 cache_id) { return -ENOENT; } + +static inline int acpi_pptt_get_cache_size_from_id(u32 cache_id, u32 *size) +{ + return -ENOENT; +} + static inline int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus) { diff --git a/include/linux/riscv_qos.h b/include/linux/riscv_qos.h new file mode 100644 index 00000000000000..1712fb12f6bc15 --- /dev/null +++ b/include/linux/riscv_qos.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __LINUX_RISCV_QOS_H +#define __LINUX_RISCV_QOS_H + +#include +#include +#include + +#include + +enum cbqri_controller_type { + CBQRI_CONTROLLER_TYPE_CAPACITY, + CBQRI_CONTROLLER_TYPE_BANDWIDTH, + CBQRI_CONTROLLER_TYPE_UNKNOWN +}; + +/* Capacity Controller hardware capabilities */ +struct riscv_cbqri_capacity_caps { + u16 ncblks; /* number of capacity blocks */ + + bool supports_alloc_at_code; + bool supports_alloc_op_flush_rcid; +}; + +/* Bandwidth Controller hardware capabilities */ +struct riscv_cbqri_bandwidth_caps { + u16 nbwblks; /* number of bandwidth blocks */ + u16 mrbwb; /* max reserved bw blocks */ + + bool supports_alloc_at_code; +}; + +struct cbqri_controller { + void __iomem *base; + /* + * Protects multi-step MMIO register sequences on this controller. + * CBQRI operations (e.g. CONFIG_LIMIT, READ_LIMIT) require writing + * an operation register, waiting for the busy flag to clear, then + * reading back the result. These sequences must be atomic per + * controller to prevent interleaving. + */ + spinlock_t lock; + + int ver_major; + int ver_minor; + + struct riscv_cbqri_bandwidth_caps bc; + struct riscv_cbqri_capacity_caps cc; + + bool alloc_capable; + + phys_addr_t addr; + phys_addr_t size; + enum cbqri_controller_type type; + u32 rcid_count; + u32 mcid_count; + struct list_head list; + + struct cache_controller { + u32 cache_level; + u32 cache_size; /* in bytes */ + struct cpumask cpu_mask; + /* Unique Cache ID from the PPTT table's Cache Type Structure */ + u32 cache_id; + } cache; + + struct mem_controller { + /* Proximity Domain from SRAT table Memory Affinity Controller */ + u32 prox_dom; + struct cpumask cpu_mask; + } mem; +}; + +extern struct list_head cbqri_controllers; + +bool resctrl_arch_alloc_capable(void); +bool resctrl_arch_mon_capable(void); + +struct rdt_resource; +/* + * Note about terminology between x86 (Intel RDT/AMD QoS) and RISC-V: + * CLOSID on x86 is RCID on RISC-V + * RMID on x86 is MCID on RISC-V + * CDP on x86 is AT (access type) on RISC-V + */ +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid); +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid); +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid); +void resctrl_arch_sched_in(struct task_struct *tsk); +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid); +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid); +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, + void *arch_mon_ctx); + +static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) +{ + return val; +} + +/* Not needed for RISC-V */ +static inline void resctrl_arch_enable_mon(void) { } +static inline void resctrl_arch_disable_mon(void) { } +static inline void resctrl_arch_enable_alloc(void) { } +static inline void resctrl_arch_disable_alloc(void) { } + +#endif /* __LINUX_RISCV_QOS_H */