diff --git a/.jules/thunderbolt.md b/.jules/thunderbolt.md index 1efe119..d5e17bc 100644 --- a/.jules/thunderbolt.md +++ b/.jules/thunderbolt.md @@ -27,3 +27,10 @@ **Evidence:** Microbenchmarking showed a 2x speedup (99ms -> 49ms) for max_v3 over max_v2 on L1-hot arrays. End-to-end framework benchmarks showed an 8% throughput increase (4.03 -> 4.36 GFLOP/s) on large fixed-memory allocations (N=6553600). **Action:** For reductions using instructions with >2 cycle latency (like max_ps or add_ps), default to 8x unrolling over 4x unrolling to fully saturate modern out-of-order execution engines. +## 2024-10-27 - AVX2 Softmax 8x Unrolling + +**Learning:** When unrolling AVX2 Softmax kernels, standard 4x unrolling leaves execution ports idle because the math instructions (especially those involved in exponential calculations and FMAs) have high enough latencies that 4 independent streams cannot fully saturate the execution engines. By unrolling 8x and maintaining 8 independent accumulators / FMA streams, the instruction latency is completely hidden, fully saturating the execution ports. + +**Evidence:** Microbenchmarking 8x unroll (`softmax_v6`) vs 4x unroll (`softmax_v5`) showed a throughput increase where the execution time dropped from ~0.96ms to ~0.92-0.93ms on L1/L2 hot data for N=1048576, transitioning the kernel from being latency-bound to being throughput/bandwidth bound. + +**Action:** For heavy math kernels (like Softmax with exponential approximations) using instructions with significant latency, default to 8x unrolling over 4x unrolling to maintain enough independent instruction streams to fully saturate modern out-of-order execution engines. diff --git a/a.out b/a.out deleted file mode 100755 index ae8a9eb..0000000 Binary files a/a.out and /dev/null differ diff --git a/ml_kernels/include/ml_kernels/softmax.h b/ml_kernels/include/ml_kernels/softmax.h index 4c6ed7a..2de51ae 100644 --- a/ml_kernels/include/ml_kernels/softmax.h +++ b/ml_kernels/include/ml_kernels/softmax.h @@ -501,4 +501,163 @@ inline void softmax_v5(const float *input, float *output, std::size_t n) { } } + +// ⚡ Thunderbolt: AVX2 Vectorized Softmax with 8x unrolling and instruction interleaving +// Target: AVX2 (Haswell+) +// Reason: 8x unrolling is needed for independent FMAs / Exps since instruction latency for them is large enough that 4x unrolling leaves execution ports idle. This fully saturates the execution ports and transitions the kernel from latency-bound to throughput-bound. +// Expected gain: ~5% throughput improvement over 4x unroll. +inline void softmax_v6(const float *input, float *output, std::size_t n) { + if (n == 0) return; + + // 1. Find max + std::size_t i = 0; + __m256 max_v = _mm256_set1_ps(std::numeric_limits::lowest()); + __m256 max0 = max_v, max1 = max_v, max2 = max_v, max3 = max_v; + __m256 max4 = max_v, max5 = max_v, max6 = max_v, max7 = max_v; + + for (; i + 63 < n; i += 64) { + max0 = _mm256_max_ps(max0, _mm256_loadu_ps(input + i)); + max1 = _mm256_max_ps(max1, _mm256_loadu_ps(input + i + 8)); + max2 = _mm256_max_ps(max2, _mm256_loadu_ps(input + i + 16)); + max3 = _mm256_max_ps(max3, _mm256_loadu_ps(input + i + 24)); + max4 = _mm256_max_ps(max4, _mm256_loadu_ps(input + i + 32)); + max5 = _mm256_max_ps(max5, _mm256_loadu_ps(input + i + 40)); + max6 = _mm256_max_ps(max6, _mm256_loadu_ps(input + i + 48)); + max7 = _mm256_max_ps(max7, _mm256_loadu_ps(input + i + 56)); + } + max0 = _mm256_max_ps(max0, max1); + max2 = _mm256_max_ps(max2, max3); + max4 = _mm256_max_ps(max4, max5); + max6 = _mm256_max_ps(max6, max7); + + max0 = _mm256_max_ps(max0, max2); + max4 = _mm256_max_ps(max4, max6); + + max0 = _mm256_max_ps(max0, max4); + + for (; i + 7 < n; i += 8) { + max0 = _mm256_max_ps(max0, _mm256_loadu_ps(input + i)); + } + float max_val = reduce_max(max0); + for (; i < n; ++i) max_val = std::max(max_val, input[i]); + + __m256 max_vec = _mm256_set1_ps(max_val); + + // 2. Compute exp and sum + i = 0; + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + __m256 sum2 = _mm256_setzero_ps(); + __m256 sum3 = _mm256_setzero_ps(); + __m256 sum4 = _mm256_setzero_ps(); + __m256 sum5 = _mm256_setzero_ps(); + __m256 sum6 = _mm256_setzero_ps(); + __m256 sum7 = _mm256_setzero_ps(); + + for (; i + 63 < n; i += 64) { + __m256 x0 = _mm256_sub_ps(_mm256_loadu_ps(input + i), max_vec); + __m256 x1 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 8), max_vec); + __m256 x2 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 16), max_vec); + __m256 x3 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 24), max_vec); + __m256 x4 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 32), max_vec); + __m256 x5 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 40), max_vec); + __m256 x6 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 48), max_vec); + __m256 x7 = _mm256_sub_ps(_mm256_loadu_ps(input + i + 56), max_vec); + + __m256 e0 = exp256_ps_v2(x0); + __m256 e1 = exp256_ps_v2(x1); + __m256 e2 = exp256_ps_v2(x2); + __m256 e3 = exp256_ps_v2(x3); + __m256 e4 = exp256_ps_v2(x4); + __m256 e5 = exp256_ps_v2(x5); + __m256 e6 = exp256_ps_v2(x6); + __m256 e7 = exp256_ps_v2(x7); + + _mm256_storeu_ps(output + i, e0); + _mm256_storeu_ps(output + i + 8, e1); + _mm256_storeu_ps(output + i + 16, e2); + _mm256_storeu_ps(output + i + 24, e3); + _mm256_storeu_ps(output + i + 32, e4); + _mm256_storeu_ps(output + i + 40, e5); + _mm256_storeu_ps(output + i + 48, e6); + _mm256_storeu_ps(output + i + 56, e7); + + sum0 = _mm256_add_ps(sum0, e0); + sum1 = _mm256_add_ps(sum1, e1); + sum2 = _mm256_add_ps(sum2, e2); + sum3 = _mm256_add_ps(sum3, e3); + sum4 = _mm256_add_ps(sum4, e4); + sum5 = _mm256_add_ps(sum5, e5); + sum6 = _mm256_add_ps(sum6, e6); + sum7 = _mm256_add_ps(sum7, e7); + } + + sum0 = _mm256_add_ps(sum0, sum1); + sum2 = _mm256_add_ps(sum2, sum3); + sum4 = _mm256_add_ps(sum4, sum5); + sum6 = _mm256_add_ps(sum6, sum7); + + sum0 = _mm256_add_ps(sum0, sum2); + sum4 = _mm256_add_ps(sum4, sum6); + + sum0 = _mm256_add_ps(sum0, sum4); + + for (; i + 7 < n; i += 8) { + __m256 x = _mm256_loadu_ps(input + i); + __m256 e = exp256_ps_v2(_mm256_sub_ps(x, max_vec)); + _mm256_storeu_ps(output + i, e); + sum0 = _mm256_add_ps(sum0, e); + } + + float sum_val = reduce_sum(sum0); + for (; i < n; ++i) { + float e = std::exp(input[i] - max_val); + output[i] = e; + sum_val += e; + } + + if (sum_val == 0.0f) return; + + // 3. Normalize + float inv_sum = 1.0f / sum_val; + __m256 inv_sum_v = _mm256_set1_ps(inv_sum); + i = 0; + + for (; i + 63 < n; i += 64) { + __m256 o0 = _mm256_loadu_ps(output + i); + __m256 o1 = _mm256_loadu_ps(output + i + 8); + __m256 o2 = _mm256_loadu_ps(output + i + 16); + __m256 o3 = _mm256_loadu_ps(output + i + 24); + __m256 o4 = _mm256_loadu_ps(output + i + 32); + __m256 o5 = _mm256_loadu_ps(output + i + 40); + __m256 o6 = _mm256_loadu_ps(output + i + 48); + __m256 o7 = _mm256_loadu_ps(output + i + 56); + + __m256 m0 = _mm256_mul_ps(o0, inv_sum_v); + __m256 m1 = _mm256_mul_ps(o1, inv_sum_v); + __m256 m2 = _mm256_mul_ps(o2, inv_sum_v); + __m256 m3 = _mm256_mul_ps(o3, inv_sum_v); + __m256 m4 = _mm256_mul_ps(o4, inv_sum_v); + __m256 m5 = _mm256_mul_ps(o5, inv_sum_v); + __m256 m6 = _mm256_mul_ps(o6, inv_sum_v); + __m256 m7 = _mm256_mul_ps(o7, inv_sum_v); + + _mm256_storeu_ps(output + i, m0); + _mm256_storeu_ps(output + i + 8, m1); + _mm256_storeu_ps(output + i + 16, m2); + _mm256_storeu_ps(output + i + 24, m3); + _mm256_storeu_ps(output + i + 32, m4); + _mm256_storeu_ps(output + i + 40, m5); + _mm256_storeu_ps(output + i + 48, m6); + _mm256_storeu_ps(output + i + 56, m7); + } + + for (; i + 7 < n; i += 8) { + _mm256_storeu_ps(output + i, _mm256_mul_ps(_mm256_loadu_ps(output + i), inv_sum_v)); + } + for (; i < n; ++i) { + output[i] *= inv_sum; + } +} + } // namespace ml_kernels diff --git a/ml_kernels/src/kernel_bench.cpp b/ml_kernels/src/kernel_bench.cpp index d22dc06..f69eb7e 100644 --- a/ml_kernels/src/kernel_bench.cpp +++ b/ml_kernels/src/kernel_bench.cpp @@ -332,6 +332,18 @@ class SoftmaxV5Benchmark : public SoftmaxBenchmark { }; REGISTER_BENCHMARK(SoftmaxV5Benchmark); +class SoftmaxV6Benchmark : public SoftmaxBenchmark { +public: + const char *name() const override { return "softmax_v6"; } + + void run() override { + ml_kernels::softmax_v6(inputs_[current_idx_].data(), outputs_[current_idx_].data(), inputs_[0].size()); + current_idx_ = (current_idx_ + 1) % pool_size_; + } +}; +REGISTER_BENCHMARK(SoftmaxV6Benchmark); + + } // namespace int main(int argc, char **argv) { diff --git a/ml_kernels/src/test_naive_ops.cpp b/ml_kernels/src/test_naive_ops.cpp index b0f27a6..bd8ed9f 100644 --- a/ml_kernels/src/test_naive_ops.cpp +++ b/ml_kernels/src/test_naive_ops.cpp @@ -181,11 +181,51 @@ void test_softmax_v5() { std::cout << "test_softmax_v5 passed!" << std::endl; } +void test_softmax_v6() { + std::cout << "Running test_softmax_v6..." << std::endl; + std::vector input = { + -2.0f, -0.5f, 1.0f, 3.0f, + 0.0f, 0.0f, 0.0f, 0.0f, + 100.0f, 100.0f, -100.0f, -100.0f, + 5.0f, -5.0f, 2.0f, -2.0f, + 1.1f, 1.2f, 1.3f, 1.4f, + -1.1f, -1.2f, -1.3f, -1.4f, + 10.0f, 20.0f, 30.0f, 40.0f, + -10.0f, -20.0f, -30.0f, -40.0f, + 1.0f, 2.0f, 3.0f, 4.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 9.0f, 10.0f, 11.0f, 12.0f, + 13.0f, 14.0f, 15.0f, 16.0f, + 17.0f, 18.0f, 19.0f, 20.0f, + 21.0f, 22.0f, 23.0f, 24.0f, + 25.0f, 26.0f, 27.0f, 28.0f, + 29.0f, 30.0f, 31.0f, 32.0f, + 33.0f, 34.0f, 35.0f, 36.0f, + 37.0f, 38.0f, 39.0f, 40.0f + }; + + std::vector output_naive(input.size(), 0.0f); + std::vector output_v6(input.size(), 0.0f); + + ml_kernels::softmax_naive(input.data(), output_naive.data(), input.size()); + ml_kernels::softmax_v6(input.data(), output_v6.data(), input.size()); + + float sum = 0.0f; + for (std::size_t i = 0; i < input.size(); ++i) { + assert(std::fabs(output_naive[i] - output_v6[i]) < 1e-4f); + sum += output_v6[i]; + } + assert(std::fabs(sum - 1.0f) < 1e-4f); + + std::cout << "test_softmax_v6 passed!" << std::endl; +} + int main() { test_relu_naive(); test_max_naive(); test_softmax_v3(); test_softmax_v4(); test_softmax_v5(); + test_softmax_v6(); std::cout << "All tests passed successfully!" << std::endl; } \ No newline at end of file