perf(qwen35): use in-place GDN state write in pure AR

server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp

@@ -10478,9 +10478,12 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
     // attn_scores: S_v * H * n_tokens * n_seqs floats
     // new_states:  S_v * S_v * H * n_seqs floats
     const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs;
-    float * attn_out_base  = (float *)dst->data;
-    float * state_out_base = (float *)dst->data + attn_score_elems;
+    float * attn_out_base = (float *)dst->data;
 
+    const bool inplace_state = ggml_get_op_params_i32(dst, 1) != 0;
+    float * state_out_base = inplace_state
+        ? (float *)src_state->data
+        : (float *)dst->data + attn_score_elems;
     const float * state_in_base = (const float *)src_state->data;
 
   //const int64_t rq1 = nev1 / neq1;
@@ -10502,9 +10505,12 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
 
         float * s_out = state_out_base + (iv3 * H + iv1) * S_v * S_v;
 
-        // copy input state into output buffer and operate in-place
+        // Copy input state into the packed output for the regular variant.
+        // The in-place variant operates directly on src_state instead.
         const float * s_in = state_in_base + (iv3 * H + iv1) * S_v * S_v;
-        memcpy(s_out, s_in, S_v * S_v * sizeof(float));
+        if (s_out != s_in) {
+            memcpy(s_out, s_in, S_v * S_v * sizeof(float));
+        }
 
         // attn output pointer for first token of this (head, seq)
         float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v;

server/src/qwen35/qwen35_backend.cpp

@@ -1,5 +1,6 @@
 #include "qwen35_backend.h"
 #include "placement/skip_park_guard.h"
+#include "qwen35/scoped_skip_props_check.h"
 #include "qwen35_dflash_target.h"
 #include "graph_builders.h"
 #include "dflash_feature_ring.h"
@@ -37,6 +38,12 @@
 
 #include "kv_quant.h"
 
+#if defined(DFLASH27B_BACKEND_CUDA) || defined(DFLASH27B_BACKEND_HIP) || defined(GGML_USE_HIP)
+extern "C" void ggml_cuda_set_skip_props_check(bool skip);
+#else
+static void ggml_cuda_set_skip_props_check(bool) {}
+#endif
+
 namespace dflash::common {
 
 namespace {
@@ -1326,7 +1333,7 @@ void Qwen35Backend::kvflash_upload_mask() {
     const size_t need = (size_t)sg_.attn_mask->ne[0] * sg_.attn_mask->ne[1];
     if (kvflash_mask_buf_.size() != need || kvflash_pager_.epoch() != kvflash_mask_epoch_) {
         kvflash_mask_buf_.assign(need, F16_NEG_INF);
-        kvflash_pager_.fill_slot_mask(kvflash_mask_buf_.data());   // q row 0
+        kvflash_pager_.fill_slot_mask(kvflash_mask_buf_.data());  // q row 0; pager caches
         kvflash_mask_epoch_ = kvflash_pager_.epoch();
     }
     // Upload before EVERY compute: the input tensor's buffer region is
@@ -1536,6 +1543,19 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen,
     }
 
     // AR decode loop for remaining tokens
+    //
+    // Graph reuse: build_target_step rebuilds the ggml cgraph every step,
+    // which resets the ggml-cuda CUDA-graph warmup counter and prevents
+    // replay. The only shape that varies across steps is the FA view window
+    // (win_len_padded = round_up(committed+1, 256)), which only advances
+    // every 256 decode steps. So we rebuild only when committed/256 crosses
+    // a boundary; within a bucket we reuse the cached graph and just update
+    // the mutable input tensors.
+    const bool ar_graph_reuse = supports_ar_graph_reuse()
+                              && std::getenv("DFLASH_AR_NO_REUSE") == nullptr
+                              && std::getenv("DFLASH_QWEN35_NO_KVPAD") == nullptr;
+    ar_decode_fa_bucket_ = -1;  // force first-step build
+
     for (int i = initial_emitted; i < n_gen; i++) {
         int32_t tok = out_tokens.back();
 
@@ -1547,17 +1567,23 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen,
         // kvflash: graph carries a slot-validity mask alongside the
         // step-invariant set_rows write; the FA span clamps to the pool.
         const bool pool = kvflash_active();
-        if (!build_target_step(sg_, w_, cache_, target_backend_,
-                               /*kv_start=*/committed, /*n_tokens=*/1,
-                               /*with_mask=*/pool, /*capture=*/false,
-                               /*capture_delta_intermediate=*/false,
-                               /*fa_window=*/0,
-                               /*last_token_logits_only=*/false,
-                               cfg_.kq_stride_pad,
-                               should_capture_moe_router(),
-                               /*kvflash_mask=*/pool,
-                               /*capture_qk=*/pool && kvflash_qk_policy_)) {
-            return false;
+
+        const int fa_bucket = committed >> 8;  // committed / 256
+        const bool need_rebuild = !ar_graph_reuse || (fa_bucket != ar_decode_fa_bucket_);
+        if (need_rebuild) {
+            if (!build_target_step(sg_, w_, cache_, target_backend_,
+                                   /*kv_start=*/committed, /*n_tokens=*/1,
+                                   /*with_mask=*/pool, /*capture=*/false,
+                                   /*capture_delta_intermediate=*/false,
+                                   /*fa_window=*/0,
+                                   /*last_token_logits_only=*/false,
+                                   cfg_.kq_stride_pad,
+                                   should_capture_moe_router(),
+                                   /*kvflash_mask=*/pool,
+                                   /*capture_qk=*/pool && kvflash_qk_policy_)) {
+                return false;
+            }
+            ar_decode_fa_bucket_ = fa_bucket;
         }
 
         // Fill kv_write_rows with this step's cache slot for set_rows:
@@ -1579,7 +1605,19 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen,
         }
         if (pool) kvflash_upload_mask();
 
-        auto st = ggml_backend_graph_compute(target_backend_, sg_.gf);
+        // Scope the props-check skip to ONLY this target graph compute. A
+        // rebuild step must run the normal property scan so ggml-cuda resets
+        // warmup and recaptures; a stable replay step skips the O(nodes) scan.
+        // The flag is cleared immediately after so nothing else in the loop
+        // (after_target_compute, kvflash_maybe_reselect — each its own
+        // CUDA-graph compute) ever runs under a stale skip.
+        ggml_status st = GGML_STATUS_SUCCESS;
+        {
+            const bool skip_this = ar_graph_reuse && !need_rebuild;
+            dflash::qwen35::ScopedSkipPropsCheck skip_props_guard(
+                &ggml_cuda_set_skip_props_check, skip_this);
+            st = ggml_backend_graph_compute(target_backend_, sg_.gf);
+        }
         if (st != GGML_STATUS_SUCCESS) return false;
 
         after_target_compute(sg_, committed, 1);

server/src/qwen35/qwen35_backend.h

@@ -133,6 +133,10 @@ class Qwen35Backend : public ModelBackend {
                                     std::vector<int32_t> & out_tokens,
                                     const DaemonIO & io);
     virtual bool should_capture_moe_router() const { return false; }
+    // AR graph-reuse + props-skip fast path is validated on the dense backend
+    // only. MoE (expert swaps + per-step host-sync) overrides this false to
+    // fall back to per-step graph rebuild.
+    virtual bool supports_ar_graph_reuse() const { return true; }
     // Hook after kvflash pool sizing, before create_target_cache: a subclass
     // may disable the pool (kvflash_tokens_=0) when it is redundant. Default no-op.
     virtual bool post_kvflash_init_gate() { return true; }
@@ -231,6 +235,8 @@ class Qwen35Backend : public ModelBackend {
     StepGraph      draft_sg_;    // draft forward
     StepGraph      proj_sg_;     // lm-head projection (remote-lm-head mode)
 
+    int ar_decode_fa_bucket_ = -1;
+
     // ── Draft feature mirror (cross-GPU feature transfer) ────────────
     DraftFeatureMirror feature_mirror_;
     DFlashDraftIpcClient remote_draft_;

server/src/qwen35/qwen35_graph_options.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "ggml.h"
+
+#include <cstdlib>
+#include <cstring>
+
+namespace dflash::common {
+
+inline bool qwen35_env_flag_is_one(const char * name) {
+    const char * value = std::getenv(name);
+    return value != nullptr && std::strcmp(value, "1") == 0;
+}
+
+inline bool qwen35_should_use_graph_wht_k_rotation(ggml_type kv_k_type) {
+    if (std::getenv("DFLASH_NO_WHT") != nullptr) {
+        return false;
+    }
+    if (kv_k_type == GGML_TYPE_TQ3_0) {
+        return false;
+    }
+    if (qwen35_env_flag_is_one("DFLASH_FORCE_WHT")) {
+        return true;
+    }
+    // q4_0 on Ampere is already fast enough without graph-level FWHT; avoiding
+    // it removes extra prefill/decode kernels in the parity path.
+    return kv_k_type != GGML_TYPE_Q4_0;
+}
+
+}  // namespace dflash::common

server/src/qwen35/qwen35_target_graph.cpp

@@ -33,6 +33,7 @@
 #include "internal.h"
 #include "delta_net_chunked.h"
 #include "kv_quant.h"
+#include "qwen35/qwen35_graph_options.h"
 #include "qwen35_ops.h"
 #include "qwen35moe_ffn.h"
 
@@ -128,10 +129,10 @@ bool create_target_cache_partial(const TargetWeights & w,
     out.kv_k_type = kv_k_type;
     out.kv_v_type = kv_v_type;
 
-    // Graph-level FWHT K-rotation (TurboQuant-style outlier spreading with
-    // standard quant types that keep fast FA kernel paths on all arches).
-    // Skip for TQ3_0 K cache — that type already applies WHT during quantization.
-    out.kv_k_rotated = (kv_k_type != GGML_TYPE_TQ3_0);
+    // Graph-level FWHT K-rotation (TurboQuant-style outlier spreading).
+    // q4_0 on Ampere skips it by default for parity with llama.cpp's leaner
+    // q4_0 path; DFLASH_FORCE_WHT=1 restores the old graph-level rotation.
+    out.kv_k_rotated = qwen35_should_use_graph_wht_k_rotation(kv_k_type);
 
     const bool needs_256_stride =
         kv_k_type == GGML_TYPE_TQ3_0 || kv_v_type == GGML_TYPE_TQ3_0;
@@ -789,7 +790,10 @@ static ggml_tensor * build_delta_net_block(
 
     // qkv_mixed currently is [conv_channels, n_tokens, n_seqs]; we need
     // [n_tokens, conv_channels, n_seqs] to concat on dim 0.
-    ggml_tensor * qkv_T = ggml_transpose(ctx, qkv_mixed);
+    // Materialize the transpose once so both concat inputs are contiguous
+    // → concat_cont fast path fires (saves 1 ggml_cont per layer vs wrapping
+    // both inputs individually).
+    ggml_tensor * qkv_T = ggml_cont(ctx, ggml_transpose(ctx, qkv_mixed));
 
     ggml_tensor * conv_input = ggml_concat(ctx, conv_states_r, qkv_T, 0);
     // conv_input: [kernel-1 + n_tokens, conv_channels, n_seqs]
@@ -864,9 +868,14 @@ static ggml_tensor * build_delta_net_block(
     q_c = ggml_l2_norm(ctx, q_c, w.rms_eps);
     k_c = ggml_l2_norm(ctx, k_c, w.rms_eps);
 
-    // Repeat Q and K from num_k_heads to num_v_heads so they match V's layout
-    // (only needed if not using the fused op's broadcast support).
-    if (num_k_heads != num_v_heads) {
+    // Repeat Q and K from num_k_heads to num_v_heads so they match V's layout.
+    // In pure AR (cap==nullptr, parent_ids==nullptr) the kernel's fastmodulo
+    // broadcast handles the head mismatch natively — skip the repeat to save
+    // one tensor allocation and a memory-bandwidth copy per SSM layer per step.
+    // In tree/capture mode keep the repeat: the DFS parent-reload in the kernel
+    // indexes intermediate states by h_idx (v-head), so Q/K must already be
+    // in the expanded [num_v_heads] layout before the kernel runs.
+    if (num_k_heads != num_v_heads && (cap != nullptr || parent_ids != nullptr)) {
         q_c = ggml_repeat_4d(ctx, q_c, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
         k_c = ggml_repeat_4d(ctx, k_c, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
     }
@@ -918,6 +927,7 @@ static ggml_tensor * build_delta_net_block(
 
     ggml_tensor * output = nullptr;
     ggml_tensor * new_state = nullptr;
+    const bool pure_ar = !use_chunked && parent_ids == nullptr && cap == nullptr && persist_inter == nullptr;
 
     if (use_chunked) {
         auto r = build_delta_net_chunked(ctx, q_c, k_c, v_c, g_tensor, beta, s);
@@ -933,12 +943,12 @@ static ggml_tensor * build_delta_net_block(
             ? ggml_gated_delta_net_tree_persist(ctx, q_c, k_c, v_c, g_tensor, beta, s, parent_ids, persist_inter)
             : ggml_gated_delta_net_tree(ctx, q_c, k_c, v_c, g_tensor, beta, s, parent_ids);
     } else {
-        // Non-tree (chain/prefill). When capture is requested, set src[7] so
-        // the kernel writes per-token intermediates directly to the persistent
-        // cache buffer — same mechanism as _tree_persist, but without tree
-        // parent_ids. Avoids the legacy result-region cpy (and the OOB it
-        // could cause if the result tensor has no embedded intermediate region).
-        result = ggml_gated_delta_net(ctx, q_c, k_c, v_c, g_tensor, beta, s);
+        // Non-tree (chain/prefill). Pure AR writes the final recurrent state
+        // in-place; capture paths keep the regular op and attach src[7] so the
+        // kernel writes per-token intermediates directly to persistent cache.
+        result = pure_ar
+            ? ggml_gated_delta_net_inplace(ctx, q_c, k_c, v_c, g_tensor, beta, s)
+            : ggml_gated_delta_net(ctx, q_c, k_c, v_c, g_tensor, beta, s);
         if (persist_inter) {
             result->src[7] = persist_inter;
         }
@@ -965,8 +975,11 @@ static ggml_tensor * build_delta_net_block(
         S_v * S_v * H_v * r_elt,
         S_v * H_v * n_seq_tokens * n_seqs * r_elt);
 
-    // Persist new_state back to cache
-    ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, ssm_state));
+    // In pure AR, ggml_gated_delta_net_inplace writes the final recurrent state
+    // directly into ssm_state, so the explicit copy is redundant.
+    if (!pure_ar) {
+        ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, ssm_state));
+    }
 
     // Expose per-step intermediate states for spec-decode rollback. The patched
     // ggml_gated_delta_net kernel appends an intermediate-states region to the
@@ -1005,12 +1018,12 @@ static ggml_tensor * build_delta_net_block(
         ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, s));
     }
 
-    // ── Gated output norm: rms_norm(output) * silu(z_4d)
+    // ── Gated output norm: rms_norm(output) * weight * silu(z_4d)
+    // fused: swiglu_split(z_4d, normed) = silu(z_4d) * normed  (1 GLU kernel vs 2)
     ggml_tensor * z_4d = ggml_reshape_4d(ctx, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
     ggml_tensor * output_n = ggml_rms_norm(ctx, rms_norm_input_f32(ctx, output), w.rms_eps);
     output_n = ggml_mul(ctx, output_n, L.ssm_norm);
-    ggml_tensor * z_silu  = ggml_silu(ctx, z_4d);
-    output_n = ggml_mul(ctx, output_n, z_silu);
+    output_n = ggml_swiglu_split(ctx, z_4d, output_n);
 
     // Reshape to [d_inner, n_tokens]
     ggml_tensor * flat = ggml_reshape_3d(ctx, output_n,

server/src/qwen35/scoped_skip_props_check.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <utility>
+
+namespace dflash::qwen35 {
+
+// Keeps ggml-cuda's thread-local graph fast path scoped to one decode request.
+template <class SetFn>
+class ScopedSkipPropsCheck {
+public:
+    ScopedSkipPropsCheck(SetFn set, bool skip) : set_(std::move(set)) { set_(skip); }
+    ~ScopedSkipPropsCheck() { set_(false); }
+
+    ScopedSkipPropsCheck(const ScopedSkipPropsCheck &) = delete;
+    ScopedSkipPropsCheck & operator=(const ScopedSkipPropsCheck &) = delete;
+
+    void set(bool skip) { set_(skip); }
+
+private:
+    SetFn set_;
+};
+
+template <class SetFn>
+ScopedSkipPropsCheck(SetFn, bool) -> ScopedSkipPropsCheck<SetFn>;
+
+}  // namespace dflash::qwen35

server/src/qwen35moe/qwen35moe_backend.h

@@ -44,6 +44,9 @@ class Qwen35MoeBackend : public Qwen35Backend {
     bool spark_wants_bootstrap() const override;
     bool spark_bootstrap_finalize(const std::string & profile_path) override;
     void after_target_compute(StepGraph & sg, int kv_start, int n_tokens) override;
+    // MoE AR is untested under graph-reuse (expert swaps + per-step host-sync);
+    // fall back to per-step rebuild instead of inheriting the dense fast path.
+    bool supports_ar_graph_reuse() const override { return false; }
 
 private:
     // All-hot placement signal for post_kvflash_init_gate(): set when

server/src/server/http_server.cpp

@@ -23,6 +23,8 @@
 #include "tool_hint.h"
 #include "common/sha1.h"
 #include "freeze_history.h"
+#include "kv_quant.h"
+#include "qwen35_graph_options.h"
 
 #ifdef DFLASH_HAS_CURL
 #include <curl/curl.h>
@@ -842,7 +844,16 @@ static std::array<uint8_t, 16> compute_disk_cache_salt(const ServerConfig & cfg)
     uint8_t tmpl_digest[20] = {};
     sha1_hash(cfg.chat_template_src.data(), cfg.chat_template_src.size(), tmpl_digest);
 
-    // Serialization: path_len(4) + path + file_size(8) + file_mtime(8) + max_ctx(4) + tmpl_digest(20).
+    // K-rotation basis: the q4_0 WHT gate flips kv_k_rotated. A .dkv snapshot
+    // written under one rotation state and restored under another is garbage
+    // attention, so fold the resolved rotation basis into the salt — rotated
+    // and unrotated snapshots then get distinct fingerprints and never cross.
+    ggml_type kv_k_t = GGML_TYPE_COUNT, kv_v_t = GGML_TYPE_COUNT;
+    dflash::resolve_kv_types(kv_k_t, kv_v_t);
+    uint8_t kv_k_rotated =
+        dflash::common::qwen35_should_use_graph_wht_k_rotation(kv_k_t) ? 1 : 0;
+
+    // Serialization: path_len(4) + path + file_size(8) + file_mtime(8) + max_ctx(4) + tmpl_digest(20) + kv_k_rotated(1).
     std::vector<uint8_t> buf;
     uint32_t plen = (uint32_t)path.size();
     buf.insert(buf.end(), (uint8_t *)&plen,        (uint8_t *)&plen        + 4);
@@ -852,6 +863,7 @@ static std::array<uint8_t, 16> compute_disk_cache_salt(const ServerConfig & cfg)
     int32_t mc = (int32_t)cfg.max_ctx;
     buf.insert(buf.end(), (uint8_t *)&mc,          (uint8_t *)&mc          + 4);
     buf.insert(buf.end(), tmpl_digest, tmpl_digest + 20);
+    buf.push_back(kv_k_rotated);
 
     uint8_t digest[20];
     sha1_hash(buf.data(), buf.size(), digest);