diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c
index 81b7682e6c6dbc..6e8ae6c66322c3 100644
--- a/arch/riscv/kernel/traps_misaligned.c
+++ b/arch/riscv/kernel/traps_misaligned.c
@@ -522,10 +522,10 @@ static bool unaligned_ctl __read_mostly;
 static void check_unaligned_access_emulated(void *arg __always_unused)
 {
 	int cpu = smp_processor_id();
-	long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
 	unsigned long tmp_var, tmp_val;
 
-	*mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
+	if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
+		return;
 
 	__asm__ __volatile__ (
 		"       "REG_L" %[tmp], 1(%[ptr])\n"
diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c
index 11c781a4de733a..bb57eb5d19dfe3 100644
--- a/arch/riscv/kernel/unaligned_access_speed.c
+++ b/arch/riscv/kernel/unaligned_access_speed.c
@@ -27,8 +27,6 @@ DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR
 static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
 static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
 
-static cpumask_t fast_misaligned_access;
-
 static u64 __maybe_unused
 measure_cycles(void (*func)(void *dst, const void *src, size_t len),
 	       void *dst, void *src, size_t len)
@@ -131,13 +129,10 @@ static int check_unaligned_access(struct page *page)
 	 * Set the value of fast_misaligned_access of a CPU. These operations
 	 * are atomic to avoid race conditions.
 	 */
-	if (ret) {
+	if (ret)
 		per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
-		cpumask_set_cpu(cpu, &fast_misaligned_access);
-	} else {
+	else
 		per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
-		cpumask_clear_cpu(cpu, &fast_misaligned_access);
-	}
 
 	return 0;
 }
@@ -192,49 +187,24 @@ static void __init check_unaligned_access_speed_all_cpus(void)
 
 DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
 
-static void modify_unaligned_access_branches(cpumask_t *mask, int weight)
+static void modify_unaligned_access_branches(const cpumask_t *mask)
 {
-	if (cpumask_weight(mask) == weight)
+	bool fast = true;
+	int cpu;
+
+	for_each_cpu(cpu, mask) {
+		if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) {
+			fast = false;
+			break;
+		}
+	}
+
+	if (fast)
 		static_branch_enable_cpuslocked(&fast_unaligned_access_speed_key);
 	else
 		static_branch_disable_cpuslocked(&fast_unaligned_access_speed_key);
 }
 
-static void set_unaligned_access_static_branches_except_cpu(int cpu)
-{
-	/*
-	 * Same as set_unaligned_access_static_branches, except excludes the
-	 * given CPU from the result. When a CPU is hotplugged into an offline
-	 * state, this function is called before the CPU is set to offline in
-	 * the cpumask, and thus the CPU needs to be explicitly excluded.
-	 */
-
-	cpumask_t fast_except_me;
-
-	cpumask_and(&fast_except_me, &fast_misaligned_access, cpu_online_mask);
-	cpumask_clear_cpu(cpu, &fast_except_me);
-
-	modify_unaligned_access_branches(&fast_except_me, num_online_cpus() - 1);
-}
-
-static void set_unaligned_access_static_branches(void)
-{
-	/*
-	 * This will be called after check_unaligned_access_all_cpus so the
-	 * result of unaligned access speed for all CPUs will be available.
-	 *
-	 * To avoid the number of online cpus changing between reading
-	 * cpu_online_mask and calling num_online_cpus, cpus_read_lock must be
-	 * held before calling this function.
-	 */
-
-	cpumask_t fast_and_online;
-
-	cpumask_and(&fast_and_online, &fast_misaligned_access, cpu_online_mask);
-
-	modify_unaligned_access_branches(&fast_and_online, num_online_cpus());
-}
-
 static int riscv_online_cpu(unsigned int cpu)
 {
 	int ret = cpu_online_unaligned_access_init(cpu);
@@ -266,14 +236,19 @@ static int riscv_online_cpu(unsigned int cpu)
 #endif
 
 exit:
-	set_unaligned_access_static_branches();
+	modify_unaligned_access_branches(cpu_online_mask);
 
 	return 0;
 }
 
 static int riscv_offline_cpu(unsigned int cpu)
 {
-	set_unaligned_access_static_branches_except_cpu(cpu);
+	cpumask_t mask;
+
+	cpumask_copy(&mask, cpu_online_mask);
+	cpumask_clear_cpu(cpu, &mask);
+
+	modify_unaligned_access_branches(&mask);
 
 	return 0;
 }
@@ -430,7 +405,7 @@ static int __init check_unaligned_access_all_cpus(void)
 				  riscv_online_cpu_vec, NULL);
 
 	cpus_read_lock();
-	set_unaligned_access_static_branches();
+	modify_unaligned_access_branches(cpu_online_mask);
 	cpus_read_unlock();
 
 	return 0;