From 79f5eaf55eb8e20be0a64cc9a5135711849bc70f Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Thu, 12 Feb 2026 16:08:12 +0100 Subject: [PATCH 1/3] Do not mutate cgraph for fused ADDs 1. We should try to minimize in-place changes to the incoming ggml_cgraph where possible (those should happen in graph_optimize) 2. Modifying in-place leads to an additional, unnecessary graph capture step as we store the properties before modifying the graph in-place in the cuda-backend --- ggml/src/ggml-cuda/ggml-cuda.cu | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index b163468789fc..7dc688483ad1 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3640,11 +3640,13 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud n_fuse++; if (n_fuse > 1) { + ggml_tensor fused_add_node; + memcpy(&fused_add_node, node, sizeof(ggml_tensor)); for (int j = 0; j < n_fuse - 1; ++j) { - node->src[j + 2] = cgraph->nodes[i + j + 1]->src[1]; + fused_add_node.src[j + 2] = cgraph->nodes[i + j + 1]->src[1]; } - cgraph->nodes[i + n_fuse - 1]->data = node->data; - ggml_cuda_op_fused_add(*cuda_ctx, node, n_fuse); + fused_add_node.data = cgraph->nodes[i + n_fuse - 1]->data; + ggml_cuda_op_fused_add(*cuda_ctx, &fused_add_node, n_fuse); i += n_fuse - 1; continue; From 17717f3fc6653196e018823b858edc673c0c913e Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Thu, 12 Feb 2026 17:36:28 +0100 Subject: [PATCH 2/3] Assert ggml_tensor is trivially copyable --- ggml/src/ggml-cuda/ggml-cuda.cu | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 7dc688483ad1..4f0a8f196dff 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3641,6 +3641,10 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud if (n_fuse > 1) { ggml_tensor fused_add_node; + // Let's ensure compilation fails should ggml_tensor at one point no-longer be trivially copyable. + static_assert( + std::is_trivially_copyable_v, + "ggml_tensor must be trivially copyable for memcpy usage in fused-add node-creation"); memcpy(&fused_add_node, node, sizeof(ggml_tensor)); for (int j = 0; j < n_fuse - 1; ++j) { fused_add_node.src[j + 2] = cgraph->nodes[i + j + 1]->src[1]; From d812b6955b8332d88aea808f48ea997995efe1e4 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Fri, 13 Feb 2026 10:34:24 +0100 Subject: [PATCH 3/3] Update ggml/src/ggml-cuda/ggml-cuda.cu Co-authored-by: Aman Gupta --- ggml/src/ggml-cuda/ggml-cuda.cu | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 4f0a8f196dff..7dc688483ad1 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3641,10 +3641,6 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud if (n_fuse > 1) { ggml_tensor fused_add_node; - // Let's ensure compilation fails should ggml_tensor at one point no-longer be trivially copyable. - static_assert( - std::is_trivially_copyable_v, - "ggml_tensor must be trivially copyable for memcpy usage in fused-add node-creation"); memcpy(&fused_add_node, node, sizeof(ggml_tensor)); for (int j = 0; j < n_fuse - 1; ++j) { fused_add_node.src[j + 2] = cgraph->nodes[i + j + 1]->src[1];