From d4e01035cc9aeece337046d663862ec76539c693 Mon Sep 17 00:00:00 2001 From: Yee Man Chan Date: Sun, 8 Feb 2026 16:19:34 +0800 Subject: [PATCH 1/7] fix vulkan ggml_acc only works in 3d but not 4d --- ggml/src/ggml-vulkan/vulkan-shaders/acc.comp | 14 +++-- tests/test-backend-ops.cpp | 59 ++++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp index 5084a70ed49f..5b44d2885a43 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp @@ -13,17 +13,19 @@ void main() { const uint offset = p.param3; const uint src1_i = idx - offset; - const uint oz = src1_i / p.nb02; - const uint oy = (src1_i - (oz * p.nb02)) / p.nb01; - const uint ox = src1_i % p.nb01; + const uint oz = src1_i / p.nb03; + const uint remy = src1_i - oz * p.nb03; + const uint oy = remy / p.nb02; + const uint remx = remy - oy * p.nb02; + const uint ox = remx / p.nb01; + const uint ow = remx % p.nb01; uint i00, i01, i02, i03; get_indices(idx, i00, i01, i02, i03); - if (ox < p.ne10 && oy < p.ne11 && oz < p.ne12) { - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + ox + oy * p.ne10 + oz * p.ne10 * p.ne11])); + if (ow < p.ne10 && ox < p.ne11 && oy < p.ne12 && oz < p.ne13) { + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + ow + ox * p.ne10 + oy * p.ne10 * p.ne11 + oz * p.ne10 * p.ne11 * p.ne12])); } else { data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); } } - diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 6fe1780f3baf..ccf9fd01a962 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5852,6 +5852,63 @@ struct test_acc : public test_case { } }; +// GGML_OP_ACC - block accumulation test +struct test_acc_block: public test_case { + const ggml_type type; + const int64_t block_size; + const int64_t n_blocks; + const int64_t ne2; + const int64_t ne3; + + std::string vars() override { + return VARS_TO_STR5(type, block_size, n_blocks, ne2, ne3); + } + + test_acc_block(ggml_type type = GGML_TYPE_F32, + int64_t block_size = 16, + int64_t n_blocks = 4, + int64_t ne2 = 1, + int64_t ne3 = 1) + : type(type), block_size(block_size), n_blocks(n_blocks), ne2(ne2), ne3(ne3) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t chunk_size = block_size * n_blocks; + + // Base tensor initialized to zero using ggml_clamp + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, chunk_size, chunk_size, ne2, ne3); + ggml_set_param(a); + ggml_set_name(a, "a"); + ggml_tensor * acc = ggml_clamp(ctx, a, 0.0f, 0.0f); + + // Source blocks that will be accumulated at different offsets + // Mimics the lower-triangular block pattern from the original code + for (int64_t j = 0; j < n_blocks; ++j) { + for (int64_t i = 0; i <= j; ++i) { + ggml_tensor * block = ggml_new_tensor_4d(ctx, type, + block_size, block_size, ne2, ne3); + ggml_set_param(block); + + char name[64]; + snprintf(name, sizeof(name), "block_%ld_%ld", (long)j, (long)i); + ggml_set_name(block, name); + + // Accumulate block at position [i*block_size, j*block_size] + // This is the same pattern as the original code: + // offset = i_start * nb[0] + j_start * nb[1] + size_t offset = (i * block_size) * ggml_type_size(type) + + (j * block_size) * (chunk_size * ggml_type_size(type)); + + acc = ggml_acc(ctx, acc, block, + acc->nb[1], acc->nb[2], acc->nb[3], + offset); + } + } + + ggml_set_name(acc, "out"); + return acc; + } +}; + // GGML_OP_PAD struct test_pad : public test_case { const ggml_type type; @@ -8130,6 +8187,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {64, 64, 320, 1})); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {9, 9, 1280, 1})); test_cases.emplace_back(new test_acc()); + test_cases.emplace_back(new test_acc_block(GGML_TYPE_F32, 16, 4, 3, 2)); + test_cases.emplace_back(new test_acc_block(GGML_TYPE_F32, 32, 4, 2, 2)); test_cases.emplace_back(new test_pad()); test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {33, 17, 2, 1}, 4, 3, true)); // circular test_cases.emplace_back(new test_pad_ext()); From 3679d220856660320ce2134b29a69bf91ff7d209 Mon Sep 17 00:00:00 2001 From: Yee Man Chan Date: Sun, 8 Feb 2026 17:11:06 +0800 Subject: [PATCH 2/7] removed clamp in test_acc_block --- tests/test-backend-ops.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index ccf9fd01a962..22f4d822f4f1 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5878,7 +5878,6 @@ struct test_acc_block: public test_case { ggml_tensor * a = ggml_new_tensor_4d(ctx, type, chunk_size, chunk_size, ne2, ne3); ggml_set_param(a); ggml_set_name(a, "a"); - ggml_tensor * acc = ggml_clamp(ctx, a, 0.0f, 0.0f); // Source blocks that will be accumulated at different offsets // Mimics the lower-triangular block pattern from the original code @@ -5898,14 +5897,14 @@ struct test_acc_block: public test_case { size_t offset = (i * block_size) * ggml_type_size(type) + (j * block_size) * (chunk_size * ggml_type_size(type)); - acc = ggml_acc(ctx, acc, block, - acc->nb[1], acc->nb[2], acc->nb[3], + a = ggml_acc(ctx, a, block, + a->nb[1], a->nb[2], a->nb[3], offset); } } - ggml_set_name(acc, "out"); - return acc; + ggml_set_name(a, "out"); + return a; } }; From 635a72a15517230f40717e9e37a9065294ad8d55 Mon Sep 17 00:00:00 2001 From: Yee Man Chan Date: Mon, 9 Feb 2026 14:13:40 +0800 Subject: [PATCH 3/7] use the correct stride and its test case --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 12 +-- ggml/src/ggml-vulkan/vulkan-shaders/acc.comp | 16 ++-- tests/test-backend-ops.cpp | 99 +++++++------------- 3 files changed, 47 insertions(+), 80 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 72097ffd0ffc..e5dcd3cbda27 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -9801,16 +9801,16 @@ static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - int nb1 = dst->op_params[0] / 4; // 4 bytes of float32 - int nb2 = dst->op_params[1] / 4; // 4 bytes of float32 - // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused - int offset = dst->op_params[3] / 4; // offset in bytes + int nb1 = dst->op_params[0] / src0_type_size; // 4 bytes of float32 + int nb2 = dst->op_params[1] / src0_type_size; // 4 bytes of float32 + int nb3 = dst->op_params[2] / src0_type_size; // 4 bytes of float32 + int offset = dst->op_params[3] / src0_type_size; // offset in bytes ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ACC, { (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t) dst->nb[3] / dst_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, 0, 0.0f, 0.0f, offset, }); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp index 5b44d2885a43..0384cd6434ba 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp @@ -13,18 +13,18 @@ void main() { const uint offset = p.param3; const uint src1_i = idx - offset; - const uint oz = src1_i / p.nb03; - const uint remy = src1_i - oz * p.nb03; - const uint oy = remy / p.nb02; - const uint remx = remy - oy * p.nb02; - const uint ox = remx / p.nb01; - const uint ow = remx % p.nb01; + const uint i3 = src1_i / p.nb03; + const uint rem2 = src1_i - i3 * p.nb03; + const uint i2 = rem2 / p.nb02; + const uint rem1 = rem2 - i2 * p.nb02; + const uint i1 = rem1 / p.nb01; + const uint i0 = rem1 % p.nb01; uint i00, i01, i02, i03; get_indices(idx, i00, i01, i02, i03); - if (ow < p.ne10 && ox < p.ne11 && oy < p.ne12 && oz < p.ne13) { - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + ow + ox * p.ne10 + oy * p.ne10 * p.ne11 + oz * p.ne10 * p.ne11 * p.ne12])); + if (i0 < p.ne10 && i1 < p.ne11 && i2 < p.ne12 && i3 < p.ne13) { + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + i0 + i1 * p.nb11 + i2 * p.nb12 + i3 * p.nb13])); } else { data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 22f4d822f4f1..9624bb86a4a6 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5826,88 +5826,52 @@ struct test_acc : public test_case { const ggml_type type; const std::array ne_a; const std::array ne_b; + const int64_t stride_dim; std::string vars() override { - return VARS_TO_STR3(type, ne_a, ne_b); + return VARS_TO_STR4(type, ne_a, ne_b, stride_dim); } test_acc(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {256, 17, 1, 1}, - std::array ne_b = {256, 16, 1, 1}) - : type(type), ne_a(ne_a), ne_b(ne_b) {} + std::array ne_a = {256, 17, 2, 3}, + std::array ne_b = {256, 16, 2, 3}, + uint64_t stride_dim = -1) + : type(type), ne_a(ne_a), ne_b(ne_b), stride_dim(stride_dim) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); ggml_set_param(a); ggml_set_name(a, "a"); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne_b.data()); - ggml_set_param(b); + ggml_tensor * b; + if (stride_dim == 1 || stride_dim == 2 || stride_dim == 3) { + // Create a larger tensor and take a view at a non-zero offset. + // This tests that the backend correctly handles b's data offset + std::array ne_b_pad = {ne_b[0], ne_b[1], ne_b[2], ne_b[3]}; + ne_b_pad[stride_dim] += 1; + ggml_tensor * b_pad = ggml_new_tensor(ctx, type, 4, ne_b_pad.data()); + ggml_set_param(b_pad); + ggml_set_name(b_pad, "b_pad"); + // View that skips the first row, so b has a non-zero byte offset + b = ggml_view_4d(ctx, b_pad, + ne_b[0], ne_b[1], ne_b[2], ne_b[3], + b_pad->nb[1], b_pad->nb[2], b_pad->nb[3], + b_pad->nb[1]); + } else { + b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + ggml_set_param(b); + } ggml_set_name(b, "b"); - ggml_tensor * out = ggml_acc(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], b->nb[1]); + // When ne_b[0] < ne_a[0], a->nb[1] != b->nb[1], so the stride + // parameters to ggml_acc don't match b's natural stride. + ggml_tensor * out = ggml_acc(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], 0); ggml_set_name(out, "out"); return out; } }; -// GGML_OP_ACC - block accumulation test -struct test_acc_block: public test_case { - const ggml_type type; - const int64_t block_size; - const int64_t n_blocks; - const int64_t ne2; - const int64_t ne3; - - std::string vars() override { - return VARS_TO_STR5(type, block_size, n_blocks, ne2, ne3); - } - - test_acc_block(ggml_type type = GGML_TYPE_F32, - int64_t block_size = 16, - int64_t n_blocks = 4, - int64_t ne2 = 1, - int64_t ne3 = 1) - : type(type), block_size(block_size), n_blocks(n_blocks), ne2(ne2), ne3(ne3) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const int64_t chunk_size = block_size * n_blocks; - - // Base tensor initialized to zero using ggml_clamp - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, chunk_size, chunk_size, ne2, ne3); - ggml_set_param(a); - ggml_set_name(a, "a"); - - // Source blocks that will be accumulated at different offsets - // Mimics the lower-triangular block pattern from the original code - for (int64_t j = 0; j < n_blocks; ++j) { - for (int64_t i = 0; i <= j; ++i) { - ggml_tensor * block = ggml_new_tensor_4d(ctx, type, - block_size, block_size, ne2, ne3); - ggml_set_param(block); - - char name[64]; - snprintf(name, sizeof(name), "block_%ld_%ld", (long)j, (long)i); - ggml_set_name(block, name); - - // Accumulate block at position [i*block_size, j*block_size] - // This is the same pattern as the original code: - // offset = i_start * nb[0] + j_start * nb[1] - size_t offset = (i * block_size) * ggml_type_size(type) - + (j * block_size) * (chunk_size * ggml_type_size(type)); - - a = ggml_acc(ctx, a, block, - a->nb[1], a->nb[2], a->nb[3], - offset); - } - } - - ggml_set_name(a, "out"); - return a; - } -}; - // GGML_OP_PAD struct test_pad : public test_case { const ggml_type type; @@ -8185,9 +8149,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {9, 9, 1280, 1})); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {64, 64, 320, 1})); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {9, 9, 1280, 1})); - test_cases.emplace_back(new test_acc()); - test_cases.emplace_back(new test_acc_block(GGML_TYPE_F32, 16, 4, 3, 2)); - test_cases.emplace_back(new test_acc_block(GGML_TYPE_F32, 32, 4, 2, 2)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, 1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, 2)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {64, 16, 2, 3}, 3)); test_cases.emplace_back(new test_pad()); test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {33, 17, 2, 1}, 4, 3, true)); // circular test_cases.emplace_back(new test_pad_ext()); From 8045d542e689f3d68d2065ed71045b24999603cf Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 9 Feb 2026 16:16:45 +0200 Subject: [PATCH 4/7] cuda : fix "supports op" condition --- ggml/src/ggml-cuda/ggml-cuda.cu | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 9e77c231c85f..2f32d2d35ad4 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4820,8 +4820,11 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_CONV_2D_DW: case GGML_OP_CONV_TRANSPOSE_2D: case GGML_OP_POOL_2D: - case GGML_OP_ACC: return true; + case GGML_OP_ACC: + // TODO: extend support like so: + //return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]); + return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); case GGML_OP_SUM: return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_TOP_K: From f9f59b08fe47b0d9545d517204a950b87439a5b5 Mon Sep 17 00:00:00 2001 From: Yee Man Chan Date: Tue, 10 Feb 2026 10:04:23 +0800 Subject: [PATCH 5/7] change src0 to src1 in ggml_vk_acc. Update acc.comp with jeffbolznv\'s suggestion except to keep the boundary check --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 2 +- ggml/src/ggml-vulkan/vulkan-shaders/acc.comp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index e5dcd3cbda27..bb7cd8cc643f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -9804,7 +9804,7 @@ static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const int nb1 = dst->op_params[0] / src0_type_size; // 4 bytes of float32 int nb2 = dst->op_params[1] / src0_type_size; // 4 bytes of float32 int nb3 = dst->op_params[2] / src0_type_size; // 4 bytes of float32 - int offset = dst->op_params[3] / src0_type_size; // offset in bytes + int offset = dst->op_params[3] / src1_type_size; // offset in bytes ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ACC, { (uint32_t)ggml_nelements(src0), diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp index 0384cd6434ba..3d61168b56f0 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp @@ -21,11 +21,10 @@ void main() { const uint i0 = rem1 % p.nb01; uint i00, i01, i02, i03; - get_indices(idx, i00, i01, i02, i03); if (i0 < p.ne10 && i1 < p.ne11 && i2 < p.ne12 && i3 < p.ne13) { - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + i0 + i1 * p.nb11 + i2 * p.nb12 + i3 * p.nb13])); + data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i0, i1, i2, i3)])); } else { - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); + data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx])); } } From 81ddff94ce9d9209bec823cc628998ca35318b4b Mon Sep 17 00:00:00 2001 From: Yee Man Chan Date: Wed, 11 Feb 2026 18:04:09 +0800 Subject: [PATCH 6/7] version without boundary check --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 46 +++++++++++--------- ggml/src/ggml-vulkan/vulkan-shaders/acc.comp | 27 +++++------- tests/test-backend-ops.cpp | 8 ++++ 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index bb7cd8cc643f..f14bf963d744 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -9796,26 +9796,6 @@ static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, }); } -static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - int nb1 = dst->op_params[0] / src0_type_size; // 4 bytes of float32 - int nb2 = dst->op_params[1] / src0_type_size; // 4 bytes of float32 - int nb3 = dst->op_params[2] / src0_type_size; // 4 bytes of float32 - int offset = dst->op_params[3] / src1_type_size; // offset in bytes - - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ACC, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, - 0, - 0.0f, 0.0f, offset, - }); -} - static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { const ggml_tensor *first_node = cgraph->nodes[node_idx]; const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; @@ -10401,6 +10381,32 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CPY, std::move(p)); } +static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + + ggml_vk_cpy(ctx, subctx, src0, dst); + + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + int nb1 = dst->op_params[0] / src0_type_size; + int nb2 = dst->op_params[1] / src0_type_size; + int nb3 = dst->op_params[2] / src0_type_size; + int offset = dst->op_params[3] / src0_type_size; + + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ACC, { + (uint32_t)ggml_nelements(src1), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], + (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2], (uint32_t)src1->ne[3], + (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t)dst->ne[0], (uint32_t)dst->ne[1], (uint32_t)dst->ne[2], (uint32_t)dst->ne[3], + (uint32_t)dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, + 0, + 0.0f, 0.0f, offset, + }); +} + static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp index 3d61168b56f0..a99e8558303e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp @@ -7,24 +7,21 @@ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; void main() { const uint idx = gl_GlobalInvocationID.x; - if (idx >= p.ne) { + if (idx >= p.ne) { // p.ne = nelements(src1) return; } - const uint offset = p.param3; - const uint src1_i = idx - offset; - const uint i3 = src1_i / p.nb03; - const uint rem2 = src1_i - i3 * p.nb03; - const uint i2 = rem2 / p.nb02; - const uint rem1 = rem2 - i2 * p.nb02; - const uint i1 = rem1 / p.nb01; - const uint i0 = rem1 % p.nb01; + const uint i3 = idx / (p.ne12 * p.ne11 * p.ne10); + const uint rem2 = idx - i3 * (p.ne12 * p.ne11 * p.ne10); + const uint i2 = rem2 / (p.ne11 * p.ne10); + const uint rem1 = rem2 - i2 * (p.ne11 * p.ne10); + const uint i1 = rem1 / p.ne10; + const uint i0 = rem1 % p.ne10; - uint i00, i01, i02, i03; + const uint dst_offset = p.param3 + i0 + i1 * p.nb01 + i2 * p.nb02 + i3 * p.nb03; - if (i0 < p.ne10 && i1 < p.ne11 && i2 < p.ne12 && i3 < p.ne13) { - data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i0, i1, i2, i3)])); - } else { - data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx])); - } + data_d[get_doffset() + dst_offset] = D_TYPE( + FLOAT_TYPE(data_a[get_aoffset() + dst_offset]) + + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i0, i1, i2, i3)]) + ); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 9624bb86a4a6..944b4b87faf9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8589,6 +8589,14 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 512, 1)); // prefill test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 1, 1)); // generate + // acc + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, 1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, 2)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {64, 16, 2, 3}, 3)); + return test_cases; } From 120a311d7bbf7852132c3f2df71ad4ba1ae5a060 Mon Sep 17 00:00:00 2001 From: Yee Man Chan Date: Thu, 12 Feb 2026 10:46:31 +0800 Subject: [PATCH 7/7] revert back to boundary check version --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 46 +++++++++----------- ggml/src/ggml-vulkan/vulkan-shaders/acc.comp | 27 +++++++----- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index f14bf963d744..e5dcd3cbda27 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -9796,6 +9796,26 @@ static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, }); } +static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + int nb1 = dst->op_params[0] / src0_type_size; // 4 bytes of float32 + int nb2 = dst->op_params[1] / src0_type_size; // 4 bytes of float32 + int nb3 = dst->op_params[2] / src0_type_size; // 4 bytes of float32 + int offset = dst->op_params[3] / src0_type_size; // offset in bytes + + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ACC, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, + 0, + 0.0f, 0.0f, offset, + }); +} + static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { const ggml_tensor *first_node = cgraph->nodes[node_idx]; const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; @@ -10381,32 +10401,6 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CPY, std::move(p)); } -static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - - ggml_vk_cpy(ctx, subctx, src0, dst); - - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - int nb1 = dst->op_params[0] / src0_type_size; - int nb2 = dst->op_params[1] / src0_type_size; - int nb3 = dst->op_params[2] / src0_type_size; - int offset = dst->op_params[3] / src0_type_size; - - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ACC, { - (uint32_t)ggml_nelements(src1), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], - (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2], (uint32_t)src1->ne[3], - (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t)dst->ne[0], (uint32_t)dst->ne[1], (uint32_t)dst->ne[2], (uint32_t)dst->ne[3], - (uint32_t)dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, - 0, - 0.0f, 0.0f, offset, - }); -} - static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp index a99e8558303e..3d61168b56f0 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp @@ -7,21 +7,24 @@ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; void main() { const uint idx = gl_GlobalInvocationID.x; - if (idx >= p.ne) { // p.ne = nelements(src1) + if (idx >= p.ne) { return; } - const uint i3 = idx / (p.ne12 * p.ne11 * p.ne10); - const uint rem2 = idx - i3 * (p.ne12 * p.ne11 * p.ne10); - const uint i2 = rem2 / (p.ne11 * p.ne10); - const uint rem1 = rem2 - i2 * (p.ne11 * p.ne10); - const uint i1 = rem1 / p.ne10; - const uint i0 = rem1 % p.ne10; + const uint offset = p.param3; + const uint src1_i = idx - offset; + const uint i3 = src1_i / p.nb03; + const uint rem2 = src1_i - i3 * p.nb03; + const uint i2 = rem2 / p.nb02; + const uint rem1 = rem2 - i2 * p.nb02; + const uint i1 = rem1 / p.nb01; + const uint i0 = rem1 % p.nb01; - const uint dst_offset = p.param3 + i0 + i1 * p.nb01 + i2 * p.nb02 + i3 * p.nb03; + uint i00, i01, i02, i03; - data_d[get_doffset() + dst_offset] = D_TYPE( - FLOAT_TYPE(data_a[get_aoffset() + dst_offset]) + - FLOAT_TYPE(data_b[get_boffset() + src1_idx(i0, i1, i2, i3)]) - ); + if (i0 < p.ne10 && i1 < p.ne11 && i2 < p.ne12 && i3 < p.ne13) { + data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i0, i1, i2, i3)])); + } else { + data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx])); + } }