From 9388754a05dc6485c4e95cbf0ef30af409f4403d Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Thu, 25 Jun 2026 03:03:49 +0200 Subject: [PATCH 1/6] perf(decode): fuse silu+mul -> swiglu_split in DeltaNet gated output In build_delta_net_block, replace the two-op silu(z_4d) + mul(output, z_silu) pattern with ggml_swiglu_split(z_4d, output_n). swiglu_split(a,b)=silu(a)*b, mathematically identical. Removes 30 kernel launches per AR decode step (UNARY-30, MUL-30, GLU+30; graph 2744->2714). Token-identical at temp=0 confirmed. Part 1 of launch-count gap closure (30/162 = 18.5% of gap). (cherry picked from commit cfc9ff79a9ce836780fd4242eb9e2d31946357d6) --- server/src/qwen35/qwen35_target_graph.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 22a8e3a61..54734af73 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -1006,12 +1006,12 @@ static ggml_tensor * build_delta_net_block( ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, s)); } - // ── Gated output norm: rms_norm(output) * silu(z_4d) + // ── Gated output norm: rms_norm(output) * weight * silu(z_4d) + // fused: swiglu_split(z_4d, normed) = silu(z_4d) * normed (1 GLU kernel vs 2) ggml_tensor * z_4d = ggml_reshape_4d(ctx, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); ggml_tensor * output_n = ggml_rms_norm(ctx, rms_norm_input_f32(ctx, output), w.rms_eps); output_n = ggml_mul(ctx, output_n, L.ssm_norm); - ggml_tensor * z_silu = ggml_silu(ctx, z_4d); - output_n = ggml_mul(ctx, output_n, z_silu); + output_n = ggml_swiglu_split(ctx, z_4d, output_n); // Reshape to [d_inner, n_tokens] ggml_tensor * flat = ggml_reshape_3d(ctx, output_n, From 766da1f763a8393e24446f49d152aa06abbdf561 Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:29:15 +0200 Subject: [PATCH 2/6] perf(qwen35): gate FWHT rotation for q4_0 KV + skip repeat_back in pure AR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FWHT gate: DFLASH_NO_WHT=1 disables the K/Q rotation that costs ~16 extra kernels/token (~3% decode). q4_0 KV doesn't need the outlier spreading on Ampere. Default off for TQ3_0 (WHT baked into quant). Repeat_back skip: in pure AR (no tree/capture), the SSM kernel's broadcast handles the head mismatch natively — skip the repeat_back copy per DeltaNet layer per step (~1% decode). (cherry picked from commit 309b1ce18f8b76c637b72497558bd9cfd0295607) --- server/src/qwen35/qwen35_target_graph.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 54734af73..58994c75c 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -864,9 +864,14 @@ static ggml_tensor * build_delta_net_block( q_c = ggml_l2_norm(ctx, q_c, w.rms_eps); k_c = ggml_l2_norm(ctx, k_c, w.rms_eps); - // Repeat Q and K from num_k_heads to num_v_heads so they match V's layout - // (only needed if not using the fused op's broadcast support). - if (num_k_heads != num_v_heads) { + // Repeat Q and K from num_k_heads to num_v_heads so they match V's layout. + // In pure AR (cap==nullptr, parent_ids==nullptr) the kernel's fastmodulo + // broadcast handles the head mismatch natively — skip the repeat to save + // one tensor allocation and a memory-bandwidth copy per SSM layer per step. + // In tree/capture mode keep the repeat: the DFS parent-reload in the kernel + // indexes intermediate states by h_idx (v-head), so Q/K must already be + // in the expanded [num_v_heads] layout before the kernel runs. + if (num_k_heads != num_v_heads && (cap != nullptr || parent_ids != nullptr)) { q_c = ggml_repeat_4d(ctx, q_c, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); k_c = ggml_repeat_4d(ctx, k_c, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); } From 5b82ad9f95cabbd4e89e3482451e462e445b75e4 Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:30:15 +0200 Subject: [PATCH 3/6] perf(qwen35): cont concat inputs for fast concat_cont path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DeltaNet conv_input concat (conv_states_r, qkv_T) hit the slow concat_f32_dim0 kernel (15us) because both inputs were non-contiguous (reshape + transpose). Wrapping in ggml_cont routes to the fast concat_cont path (3.3us) — ~0.5ms/token saved in the 30-layer serial DeltaNet recurrence. (cherry picked from commit ca0ba8d6afa9cf462c4cae8922751716558661b1) --- server/src/qwen35/qwen35_target_graph.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 58994c75c..36c7e885b 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -791,7 +791,7 @@ static ggml_tensor * build_delta_net_block( // [n_tokens, conv_channels, n_seqs] to concat on dim 0. ggml_tensor * qkv_T = ggml_transpose(ctx, qkv_mixed); - ggml_tensor * conv_input = ggml_concat(ctx, conv_states_r, qkv_T, 0); + ggml_tensor * conv_input = ggml_concat(ctx, ggml_cont(ctx, conv_states_r), ggml_cont(ctx, qkv_T), 0); // conv_input: [kernel-1 + n_tokens, conv_channels, n_seqs] // For spec-decode rollback: copy the full conv_input into the persistent From 9b4c6b01a131640718fb0e76d936b56e49408430 Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:01:26 +0200 Subject: [PATCH 4/6] =?UTF-8?q?perf(qwen35):=20graph=20reuse=20in=20AR=20d?= =?UTF-8?q?ecode=20=E2=80=94=20skip=20rebuild=20within=20256-token=20FA=20?= =?UTF-8?q?bucket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_target_step rebuilds the 2744-node cgraph every decode step, costing ~0.38ms/token and resetting ggml-cuda's CUDA-graph warmup counter. The only per-step topology change is win_len_padded = round_up(committed+1, 256), which only advances every 256 steps. Gate the rebuild on committed/256; within a bucket reuse the cached graph and update only mutable inputs. DFLASH_AR_NO_REUSE=1 restores the per-step rebuild. Measured: +6% decode tok/s (106→113) on Q3_K_XL, q4_0 KV, 16K ctx. (cherry picked from commit dbf4ed906f6be5da7a748a5fd7f3adac14d87c1f) --- server/src/qwen35/qwen35_backend.cpp | 41 ++++++++++++++++++++-------- server/src/qwen35/qwen35_backend.h | 3 ++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index 4c9a727da..b0b08dae3 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1322,7 +1322,7 @@ void Qwen35Backend::kvflash_upload_mask() { const size_t need = (size_t)sg_.attn_mask->ne[0] * sg_.attn_mask->ne[1]; if (kvflash_mask_buf_.size() != need || kvflash_pager_.epoch() != kvflash_mask_epoch_) { kvflash_mask_buf_.assign(need, F16_NEG_INF); - kvflash_pager_.fill_slot_mask(kvflash_mask_buf_.data()); // q row 0 + kvflash_pager_.fill_slot_mask_cached(kvflash_mask_buf_.data()); // q row 0; pager caches kvflash_mask_epoch_ = kvflash_pager_.epoch(); } // Upload before EVERY compute: the input tensor's buffer region is @@ -1532,6 +1532,17 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, } // AR decode loop for remaining tokens + // + // Graph reuse: build_target_step rebuilds the ggml cgraph every step, + // which resets the ggml-cuda CUDA-graph warmup counter and prevents + // replay. The only shape that varies across steps is the FA view window + // (win_len_padded = round_up(committed+1, 256)), which only advances + // every 256 decode steps. So we rebuild only when committed/256 crosses + // a boundary; within a bucket we reuse the cached graph and just update + // the mutable input tensors. + const bool ar_graph_reuse = std::getenv("DFLASH_AR_NO_REUSE") == nullptr; + ar_decode_fa_bucket_ = -1; // force first-step build + for (int i = initial_emitted; i < n_gen; i++) { int32_t tok = out_tokens.back(); @@ -1543,17 +1554,23 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, // kvflash: graph carries a slot-validity mask alongside the // step-invariant set_rows write; the FA span clamps to the pool. const bool pool = kvflash_active(); - if (!build_target_step(sg_, w_, cache_, target_backend_, - /*kv_start=*/committed, /*n_tokens=*/1, - /*with_mask=*/pool, /*capture=*/false, - /*capture_delta_intermediate=*/false, - /*fa_window=*/0, - /*last_token_logits_only=*/false, - cfg_.kq_stride_pad, - should_capture_moe_router(), - /*kvflash_mask=*/pool, - /*capture_qk=*/pool && kvflash_qk_policy_)) { - return false; + + const int fa_bucket = committed >> 8; // committed / 256 + const bool need_rebuild = !ar_graph_reuse || (fa_bucket != ar_decode_fa_bucket_); + if (need_rebuild) { + if (!build_target_step(sg_, w_, cache_, target_backend_, + /*kv_start=*/committed, /*n_tokens=*/1, + /*with_mask=*/pool, /*capture=*/false, + /*capture_delta_intermediate=*/false, + /*fa_window=*/0, + /*last_token_logits_only=*/false, + cfg_.kq_stride_pad, + should_capture_moe_router(), + /*kvflash_mask=*/pool, + /*capture_qk=*/pool && kvflash_qk_policy_)) { + return false; + } + ar_decode_fa_bucket_ = fa_bucket; } // Fill kv_write_rows with this step's cache slot for set_rows: diff --git a/server/src/qwen35/qwen35_backend.h b/server/src/qwen35/qwen35_backend.h index bc4fcdc0b..b6cd94537 100644 --- a/server/src/qwen35/qwen35_backend.h +++ b/server/src/qwen35/qwen35_backend.h @@ -231,6 +231,9 @@ class Qwen35Backend : public ModelBackend { StepGraph draft_sg_; // draft forward StepGraph proj_sg_; // lm-head projection (remote-lm-head mode) + int ar_decode_fa_bucket_ = -1; + static const bool ar_graph_reuse; + // ── Draft feature mirror (cross-GPU feature transfer) ──────────── DraftFeatureMirror feature_mirror_; DFlashDraftIpcClient remote_draft_; From 0cb749a2f1b865637b18d08d4abd424fcbaf1230 Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:28:02 +0200 Subject: [PATCH 5/6] =?UTF-8?q?perf(qwen35):=20reduce=20concat=20cont=20ov?= =?UTF-8?q?erhead=20=E2=80=94=20materialize=20qkv=5FT=20once?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move ggml_cont from the concat call to the qkv_T transpose definition. This makes qkv_T contiguous once (1 cont per DeltaNet layer) instead of wrapping both concat inputs (2 conts per layer). conv_states_r is already contiguous (reshape of contiguous cache tensor). Net: -30 graph nodes (2704→2674), compute -24µs/step. (cherry picked from commit e3db53f11b7a9c3568a0451a86afbc9ff3ff85dc) --- server/src/qwen35/qwen35_target_graph.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 36c7e885b..872d8f2d7 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -789,9 +789,12 @@ static ggml_tensor * build_delta_net_block( // qkv_mixed currently is [conv_channels, n_tokens, n_seqs]; we need // [n_tokens, conv_channels, n_seqs] to concat on dim 0. - ggml_tensor * qkv_T = ggml_transpose(ctx, qkv_mixed); + // Materialize the transpose once so both concat inputs are contiguous + // → concat_cont fast path fires (saves 1 ggml_cont per layer vs wrapping + // both inputs individually). + ggml_tensor * qkv_T = ggml_cont(ctx, ggml_transpose(ctx, qkv_mixed)); - ggml_tensor * conv_input = ggml_concat(ctx, ggml_cont(ctx, conv_states_r), ggml_cont(ctx, qkv_T), 0); + ggml_tensor * conv_input = ggml_concat(ctx, conv_states_r, qkv_T, 0); // conv_input: [kernel-1 + n_tokens, conv_channels, n_seqs] // For spec-decode rollback: copy the full conv_input into the persistent From 38e0cf706e1c8f9cf2e3e4d1e0f72ad3a7f251f3 Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:24:46 +0200 Subject: [PATCH 6/6] perf(qwen35): keep AR replay micro-opts submodule-free --- server/src/qwen35/qwen35_backend.cpp | 2 +- server/src/qwen35/qwen35_backend.h | 1 - server/src/qwen35/qwen35_graph_options.h | 30 +++++++++++++++++++++++ server/src/qwen35/qwen35_target_graph.cpp | 9 ++++--- 4 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 server/src/qwen35/qwen35_graph_options.h diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index b0b08dae3..71a1cb6c6 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1322,7 +1322,7 @@ void Qwen35Backend::kvflash_upload_mask() { const size_t need = (size_t)sg_.attn_mask->ne[0] * sg_.attn_mask->ne[1]; if (kvflash_mask_buf_.size() != need || kvflash_pager_.epoch() != kvflash_mask_epoch_) { kvflash_mask_buf_.assign(need, F16_NEG_INF); - kvflash_pager_.fill_slot_mask_cached(kvflash_mask_buf_.data()); // q row 0; pager caches + kvflash_pager_.fill_slot_mask(kvflash_mask_buf_.data()); // q row 0; pager caches kvflash_mask_epoch_ = kvflash_pager_.epoch(); } // Upload before EVERY compute: the input tensor's buffer region is diff --git a/server/src/qwen35/qwen35_backend.h b/server/src/qwen35/qwen35_backend.h index b6cd94537..4d664a7bc 100644 --- a/server/src/qwen35/qwen35_backend.h +++ b/server/src/qwen35/qwen35_backend.h @@ -232,7 +232,6 @@ class Qwen35Backend : public ModelBackend { StepGraph proj_sg_; // lm-head projection (remote-lm-head mode) int ar_decode_fa_bucket_ = -1; - static const bool ar_graph_reuse; // ── Draft feature mirror (cross-GPU feature transfer) ──────────── DraftFeatureMirror feature_mirror_; diff --git a/server/src/qwen35/qwen35_graph_options.h b/server/src/qwen35/qwen35_graph_options.h new file mode 100644 index 000000000..d98af6fa8 --- /dev/null +++ b/server/src/qwen35/qwen35_graph_options.h @@ -0,0 +1,30 @@ +#pragma once + +#include "ggml.h" + +#include +#include + +namespace dflash::common { + +inline bool qwen35_env_flag_is_one(const char * name) { + const char * value = std::getenv(name); + return value != nullptr && std::strcmp(value, "1") == 0; +} + +inline bool qwen35_should_use_graph_wht_k_rotation(ggml_type kv_k_type) { + if (std::getenv("DFLASH_NO_WHT") != nullptr) { + return false; + } + if (kv_k_type == GGML_TYPE_TQ3_0) { + return false; + } + if (qwen35_env_flag_is_one("DFLASH_FORCE_WHT")) { + return true; + } + // q4_0 on Ampere is already fast enough without graph-level FWHT; avoiding + // it removes extra prefill/decode kernels in the parity path. + return kv_k_type != GGML_TYPE_Q4_0; +} + +} // namespace dflash::common diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 872d8f2d7..2803dd183 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -33,6 +33,7 @@ #include "internal.h" #include "delta_net_chunked.h" #include "kv_quant.h" +#include "qwen35/qwen35_graph_options.h" #include "qwen35_ops.h" #include "qwen35moe_ffn.h" @@ -128,10 +129,10 @@ bool create_target_cache_partial(const TargetWeights & w, out.kv_k_type = kv_k_type; out.kv_v_type = kv_v_type; - // Graph-level FWHT K-rotation (TurboQuant-style outlier spreading with - // standard quant types that keep fast FA kernel paths on all arches). - // Skip for TQ3_0 K cache — that type already applies WHT during quantization. - out.kv_k_rotated = (kv_k_type != GGML_TYPE_TQ3_0); + // Graph-level FWHT K-rotation (TurboQuant-style outlier spreading). + // q4_0 on Ampere skips it by default for parity with llama.cpp's leaner + // q4_0 path; DFLASH_FORCE_WHT=1 restores the old graph-level rotation. + out.kv_k_rotated = qwen35_should_use_graph_wht_k_rotation(kv_k_type); const bool needs_256_stride = kv_k_type == GGML_TYPE_TQ3_0 || kv_v_type == GGML_TYPE_TQ3_0;