From 9669388584ce3e79ba3e8d9123b112957014c95d Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 8 Apr 2026 17:40:34 +0200 Subject: [PATCH 1/2] CUDA: also store node->src->data ptrs for equality check --- ggml/src/ggml-cuda/common.cuh | 6 +++++- ggml/src/ggml-cuda/ggml-cuda.cu | 21 ++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index a2960e5ae3ce..f74cb37d24a7 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1173,7 +1173,11 @@ struct ggml_cuda_graph { std::vector nodes; bool disable_due_to_gpu_arch = false; bool warmup_complete = false; - std::vector nodes_copy; + struct node_property { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + }; + std::vector nodes_props; bool is_enabled() const { static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index b21196bb4f36..2b71bc92df5c 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2979,18 +2979,25 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); // Check if the graph size has changed - if ((int)graph->nodes_copy.size() != cgraph->n_nodes) { + if ((int)graph->nodes_props.size() != cgraph->n_nodes) { res = true; - graph->nodes_copy.resize(cgraph->n_nodes); + graph->nodes_props.resize(cgraph->n_nodes); } for (int i = 0; i < cgraph->n_nodes; i++) { - if (!res) { - if (memcmp(&graph->nodes_copy[i], cgraph->nodes[i], sizeof(ggml_tensor)) != 0) { - res = true; - } + ggml_cuda_graph::node_property prop = {}; + memcpy(&prop.node, cgraph->nodes[i], sizeof(ggml_tensor)); + + // src->data being same is enough to guarantee that node->srcs are consistent. ref: + // https://github.com/ggml-org/llama.cpp/pull/21472#discussion_r3052235188 + for (int j = 0; j < GGML_MAX_SRC; ++j) { + prop.node_src_data_ptrs[j] = cgraph->nodes[i]->src[j] ? cgraph->nodes[i]->src[j]->data : nullptr; + } + + if (!res && memcmp(&graph->nodes_props[i], &prop, sizeof(prop)) != 0) { + res = true; } - memcpy(&graph->nodes_copy[i], cgraph->nodes[i], sizeof(ggml_tensor)); + graph->nodes_props[i] = prop; } return res; From a00ef861e90e0906a50deb28ae2ff656eb2e8734 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 8 Apr 2026 18:13:02 +0200 Subject: [PATCH 2/2] address review comments --- ggml/src/ggml-cuda/common.cuh | 4 ++-- ggml/src/ggml-cuda/ggml-cuda.cu | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index f74cb37d24a7..65d7a6e22aed 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1173,11 +1173,11 @@ struct ggml_cuda_graph { std::vector nodes; bool disable_due_to_gpu_arch = false; bool warmup_complete = false; - struct node_property { + struct node_properties { ggml_tensor node; void * node_src_data_ptrs[GGML_MAX_SRC]; }; - std::vector nodes_props; + std::vector node_props; bool is_enabled() const { static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 2b71bc92df5c..648124c0d31e 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2979,25 +2979,25 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); // Check if the graph size has changed - if ((int)graph->nodes_props.size() != cgraph->n_nodes) { + if ((int)graph->node_props.size() != cgraph->n_nodes) { res = true; - graph->nodes_props.resize(cgraph->n_nodes); + graph->node_props.resize(cgraph->n_nodes); } for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_cuda_graph::node_property prop = {}; + ggml_cuda_graph::node_properties prop = {}; memcpy(&prop.node, cgraph->nodes[i], sizeof(ggml_tensor)); - // src->data being same is enough to guarantee that node->srcs are consistent. ref: + // if the backend scheduler is making copies of CPU tensors, the src pointers can be the same but with different data, see: // https://github.com/ggml-org/llama.cpp/pull/21472#discussion_r3052235188 for (int j = 0; j < GGML_MAX_SRC; ++j) { prop.node_src_data_ptrs[j] = cgraph->nodes[i]->src[j] ? cgraph->nodes[i]->src[j]->data : nullptr; } - if (!res && memcmp(&graph->nodes_props[i], &prop, sizeof(prop)) != 0) { + if (!res && memcmp(&graph->node_props[i], &prop, sizeof(prop)) != 0) { res = true; } - graph->nodes_props[i] = prop; + graph->node_props[i] = prop; } return res;