From b9c006970bf863d44ff186f0b23c02a828dd3bc8 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:41:07 +0200 Subject: [PATCH 1/2] feat(ggml): vendor ROCmFPX quant support for DeepSeek V4 Flash Ports the ROCmFP4/ROCmFPX ggml work (previously lucebox-ggml #36, on the old submodule pointer) into the vendored server/deps/llama.cpp tree. - ROCmFP4/ROCmFPX quant types + CPU reference conversions (ggml/rocmfp4, ggml/rocmfpx) and the ggml type-trait registrations. - CUDA/HIP dequant, copy, getrows, MMVQ vecdot, MMVF, unary and FA paths for the new types. - Fused DS4 hyper-connection op (GGML_OP_DS4_HC) with the register-resident sinkhorn kernel; inert unless emitted by the DS4 fused-decode path. - DS4 SwiGLU split op plumbing for the fused FFN matvec paths. Layered on current main, so the main-side ggml work is preserved (fp64 RoPE reduction, #497 RDNA MMQ tile, LUCE_MMQ_DP_MAX_NE1, MMVQ_MAX_MOE_BATCH_SIZE, fused dual set_rows, raw-span guard). Review fixes on top of #36: - ggml_ftype_to_ggml_type: handle the 11 new ROCmFPX ftypes (dominant-type mapping) so the enum switch is -Wswitch/-Werror clean. - FP6 MMVQ vecdot: pad qs[] to avoid a stack over-read of the last window (bit-identical; the over-read bits were already masked out). --- server/deps/llama.cpp/ggml/include/ggml.h | 60 +- server/deps/llama.cpp/ggml/rocmfp4/README.md | 1020 +++++++++++++++ server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.c | 797 ++++++++++++ server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.h | 58 + .../llama.cpp/ggml/rocmfp4/rocmfp4_hip.cu | 85 ++ .../ggml/rocmfp4/rocmfp4_hip_codebook.cuh | 79 ++ .../ggml/rocmfp4/rocmfp4_hip_scale.cuh | 157 +++ server/deps/llama.cpp/ggml/rocmfpx/README.md | 106 ++ server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c | 1140 +++++++++++++++++ server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h | 111 ++ .../llama.cpp/ggml/rocmfpx/test_rocmfpx.c | 164 +++ server/deps/llama.cpp/ggml/src/CMakeLists.txt | 4 + server/deps/llama.cpp/ggml/src/ggml-common.h | 6 + .../llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c | 4 + .../deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp | 78 ++ .../llama.cpp/ggml/src/ggml-cuda/common.cuh | 68 + .../llama.cpp/ggml/src/ggml-cuda/concat.cu | 97 +- .../llama.cpp/ggml/src/ggml-cuda/convert.cu | 110 ++ .../deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu | 131 ++ .../ggml/src/ggml-cuda/dequantize.cuh | 123 ++ .../llama.cpp/ggml/src/ggml-cuda/ds4-hc.cu | 367 ++++++ .../llama.cpp/ggml/src/ggml-cuda/ds4-hc.cuh | 10 + .../llama.cpp/ggml/src/ggml-cuda/fattn.cu | 136 +- .../llama.cpp/ggml/src/ggml-cuda/getrows.cu | 24 + .../llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu | 48 +- .../deps/llama.cpp/ggml/src/ggml-cuda/mmvf.cu | 14 +- .../deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu | 73 +- .../llama.cpp/ggml/src/ggml-cuda/unary.cu | 57 + .../llama.cpp/ggml/src/ggml-cuda/unary.cuh | 10 + .../llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh | 375 ++++++ server/deps/llama.cpp/ggml/src/ggml.c | 179 ++- 31 files changed, 5636 insertions(+), 55 deletions(-) create mode 100644 server/deps/llama.cpp/ggml/rocmfp4/README.md create mode 100644 server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.c create mode 100644 server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.h create mode 100644 server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip.cu create mode 100644 server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh create mode 100644 server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_scale.cuh create mode 100644 server/deps/llama.cpp/ggml/rocmfpx/README.md create mode 100644 server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c create mode 100644 server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h create mode 100644 server/deps/llama.cpp/ggml/rocmfpx/test_rocmfpx.c create mode 100644 server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cu create mode 100644 server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cuh diff --git a/server/deps/llama.cpp/ggml/include/ggml.h b/server/deps/llama.cpp/ggml/include/ggml.h index d76166f16..1d8f22bb8 100644 --- a/server/deps/llama.cpp/ggml/include/ggml.h +++ b/server/deps/llama.cpp/ggml/include/ggml.h @@ -430,7 +430,13 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_TQ3_0 = 42, // TurboQuant 3.5 bpv (3-bit Lloyd-Max + FWHT rotation) - GGML_TYPE_COUNT = 43, + GGML_TYPE_Q4_0_ROCMFP4 = 100, + GGML_TYPE_Q4_0_ROCMFP4_FAST = 101, + GGML_TYPE_Q6_0_ROCMFPX = 102, + GGML_TYPE_Q8_0_ROCMFPX = 103, + GGML_TYPE_Q3_0_ROCMFPX = 104, + GGML_TYPE_Q2_0_ROCMFP2 = 107, + GGML_TYPE_COUNT = 108, }; // precision @@ -468,6 +474,18 @@ extern "C" { GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4 = 100, + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_LEAN = 101, + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_COHERENT = 102, + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_FAST = 103, + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_FAST_COHERENT = 104, + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_STRIX = 105, + GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_STRIX_LEAN = 106, + GGML_FTYPE_MOSTLY_Q6_0_ROCMFPX = 110, + GGML_FTYPE_MOSTLY_Q8_0_ROCMFPX = 111, + GGML_FTYPE_MOSTLY_Q3_0_ROCMFPX = 112, + GGML_FTYPE_MOSTLY_Q2_0_ROCMFP2 = 118, + GGML_FTYPE_MOSTLY_Q2_0_ROCMFP2_STRIX = 119, }; // available tensor operations: @@ -583,6 +601,8 @@ extern "C" { GGML_OP_MOE_FUSED, // Fused MoE FFN: gate+up+swiglu+down+weighted_sum+shared_expert + GGML_OP_DS4_HC, // Fused DeepSeek4 hyper-connection pre/post/out mixing + GGML_OP_COUNT, }; @@ -618,6 +638,7 @@ extern "C" { GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU, GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_SWIGLU_DS4, GGML_GLU_OP_GEGLU_ERF, GGML_GLU_OP_GEGLU_QUICK, @@ -1339,6 +1360,12 @@ extern "C" { struct ggml_tensor * a, struct ggml_tensor * b); + GGML_API struct ggml_tensor * ggml_swiglu_ds4_split( + struct ggml_context * ctx, + struct ggml_tensor * gate, + struct ggml_tensor * up, + float clamp); + GGML_API struct ggml_tensor * ggml_geglu_erf_split( struct ggml_context * ctx, struct ggml_tensor * a, @@ -2399,6 +2426,37 @@ extern "C" { struct ggml_tensor * experts, struct ggml_tensor * expert_weights); + // Fused DeepSeek4 hyper-connection helpers (decode, n_tokens == 1). + // ggml_ds4_hc_pre: mix[2*n_hc+n_hc^2] + base + hc_state[n_embd*n_hc] -> + // dst[n_embd + 2*n_hc + n_hc^2] = { working, split(pre,post,comb) } + GGML_API struct ggml_tensor * ggml_ds4_hc_pre( + struct ggml_context * ctx, + struct ggml_tensor * mix, + struct ggml_tensor * base, + struct ggml_tensor * hc_state, + int n_hc, + int sinkhorn_iters, + float pre_scale, + float post_scale, + float comb_scale); + + // ggml_ds4_hc_post: residual hc_state + block_out + split -> new hc_state + GGML_API struct ggml_tensor * ggml_ds4_hc_post( + struct ggml_context * ctx, + struct ggml_tensor * residual_hc, + struct ggml_tensor * block_out, + struct ggml_tensor * split, + int n_hc); + + // ggml_ds4_hc_out: output-stage merge of hc streams into one embedding + GGML_API struct ggml_tensor * ggml_ds4_hc_out( + struct ggml_context * ctx, + struct ggml_tensor * mix, + struct ggml_tensor * base, + struct ggml_tensor * hc_state, + int n_hc, + float pre_scale); + // TODO: needs to be adapted to ggml_flash_attn_ext GGML_API struct ggml_tensor * ggml_flash_attn_back( struct ggml_context * ctx, diff --git a/server/deps/llama.cpp/ggml/rocmfp4/README.md b/server/deps/llama.cpp/ggml/rocmfp4/README.md new file mode 100644 index 000000000..39bb79302 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfp4/README.md @@ -0,0 +1,1020 @@ +# ROCmFP4 + +ROCmFP4 is the experimental `Q4_0_ROCMFP4` GGUF tensor type for Strix Halo. +The current AMD-tuned variant stores 32 weights per block as packed +E2M1-derived 4-bit values plus two unsigned E4M3 scale bytes, one per +16-weight half, for 18 bytes per block. The codebook stores half-scale signed +integer levels up to `10`, representing `5.0` raw-scale units after the scale +factor is applied. This keeps outlier pull lower than the original wider test +range while preserving a fast integer-dot backend shape. + +This directory owns the format-specific implementation. The rest of ggml only +registers and dispatches the type. + +Current status: +- The format runs on CPU, Vulkan, and ROCm/HIP in this custom tree. +- `Q4_0_ROCMFP4` is the pure 4.50 BPW dual-scale path. +- `Q4_0_ROCMFP4_LEAN` keeps ROCmFP4 for dense tensors but protects token + embeddings with `Q5_K`. On the Qwen3-4B Strix test pass this closed most of + the Q4_0 coherence gap while staying around normal Q4 size. +- `Q4_0_ROCMFP4_COHERENT` protects token embeddings with `Q6_K` and is the + quality-first ROCmFP4 preset. +- `Q4_0_ROCMFP4_FAST` is the 4.25 BPW single-scale speed path. It is the + smallest and fastest decode variant, but the pure version gives up too much + PPL to be the default quality target. +- `Q4_0_ROCMFP4_FAST_COHERENT` combines the fast 4.25 BPW transformer layout + with `Q6_K` token embeddings. On the Qwen3-4B Strix test pass it is the + current balanced AMD target: smaller than `Q4_0`, faster than `Q4_0` on + decode on both Vulkan and ROCm, and close to `Q4_0` PPL on the short + WikiText-2 check. +- `Q4_0_ROCMFP4_STRIX` is the current quality-biased Strix Halo preset. It + keeps most transformer tensors on `Q4_0_ROCMFP4_FAST`, protects token + embeddings with `Q6_K`, and uses the dual-scale `Q4_0_ROCMFP4` layout for + attention-K and attention-V tensors. On Qwen3-4B it improved the short + WikiText-2 PPL to `13.8865` at `4.49 BPW` while still beating the + same-flags stock `Q4_0` decode baselines on both Vulkan and ROCm. +- `Q4_0_ROCMFP4_STRIX_LEAN` is the compact Strix Halo preset. It keeps the + STRIX all-layer dual-scale attention-K/V protection, uses the FAST + single-scale transformer layout for the dense tensors, and protects token + embeddings/output with `Q5_K` instead of `Q6_K`. On the Qwen3-4B validation + pass it landed at `4.38 BPW`, improved short WikiText-2 PPL versus + `FAST_COHERENT`, and kept Vulkan decode in the `81 tok/s` band. +- A smaller first/last-layer-only K/V protection recipe was tested but not + promoted. It reached `4.48 BPW`, `80.13` Vulkan decode, and `69.85` ROCm + decode, but PPL regressed to `14.0167`, so the all-layer K/V STRIX preset + remains the quality target. +- The ROCm/HIP MMQ path for `Q4_0_ROCMFP4_FAST` uses one scale per 32-weight + block, matching the actual FAST layout instead of duplicating the scale into + two half-block slots. +- The ROCm/HIP vector-dot and MMQ loaders use a ROCmFP4-owned Codebook10 + expander backed by AMD `amdgcn_perm` constants. This avoids the generic + table-load helper on the hot ROCm path. +- The ROCm/HIP hot paths use a ROCmFP4-owned unaligned 32-bit quant-byte load + for packed nibble bytes. ROCmFP4 blocks are 17 or 18 bytes wide, so the older + byte-safe assembly path was conservative but expensive on Strix Halo HIP. + Direct unaligned dword loads improved ROCm `MUL_MAT` and FlashAttention + microbenchmarks while the Qwen3.6 27B MTP guard held `33.4 tok/s` short and + `27.7 tok/s` sustained. It is enabled by default and can be disabled with + `-DGGML_ROCMFP4_UNALIGNED_QS_DWORD_LOAD=0` for isolation. +- The ROCm/HIP FAST MMVQ/MMQ path now uses the same ROCmFP4-owned unaligned + quant-byte load instead of the generic byte-assembly helper. The full + promoted gate measured FAST ROCm `MUL_MAT` at `45.17`, `58.38`, `90.54`, + and `157.83` us for `n=1/2/4/8`, and the Qwen3.6 27B MTP guard improved to + `33.6 tok/s` short and `28.0 tok/s` sustained. The ROCm runtime guard now + tightens the FAST ceilings to protect this band. +- Qwen3.6 35B A3B MTP ROCmFP4 STRIX_LEAN was checked separately at native + `262144` context on ROCm0. In the reasoning-off discovery sweep, `n-max 1` + was best sustained at `72.2 tok/s`, while `n-max 5` was best burst at + `107.3 tok/s` but slower sustained at `64.9 tok/s`. With reasoning on, a + follow-up sweep found `n-max 2` best sustained at `92.6 tok/s` short and + `80.6 tok/s` sustained; `n-max 3` was close at `104.3` short and `80.1` + sustained, while `n-max 5` stayed burst-only at `98.6` short and `73.3` + sustained. KV-cache isolation then found that q8 main KV with q4 draft KV + improved the reasoning-on `n-max 2` profile to `93.7 tok/s` short and + `85.6 tok/s` sustained; full q8 main/draft KV reached `93.2` / `85.2`, + draft-only q8 stayed at `80.5` sustained, and K-only / V-only q8 main KV + regressed to `71.8` / `73.3` sustained. Rechecking draft depth under q8 main + KV moved the best sustained profile to `n-max 3` at `104.3 tok/s` short and + `89.3 tok/s` sustained, while `n-max 4` became the best burst-only profile + at `111.2 tok/s` short and `78.7 tok/s` sustained. The Pi sustained profile + therefore uses `n-max 3` with q8 main KV and q4 draft KV, while a separate + `n-max 4` burst profile is available for short-response experimentation. + The older `n-max 5` burst alias remains available for comparison. + Batch and CPU thread follow-ups did not beat the promoted runtime shape: + `-b 1024 -ub 512`, `-b 2048 -ub 512`, `-b 512 -ub 256`, + `-t 24 -tb 32`, and `-t 12 -tb 32` all measured in the `88.7`-`89.1 tok/s` + sustained band, so `-b 512 -ub 512 -t 16 -tb 32` stays promoted. + Current KV isolation rechecked the promoted shape at `104.3 tok/s` short and + `90.1 tok/s` sustained with q8 main KV, q4 draft KV, `n-max 3`, and + `p-min 0.25`; full q8 draft KV tied at `104.5` / `90.0`, while draft-only + q8 reached only `82.2` sustained. K-only and V-only q8 main KV regressed to + `70.5` and `74.6` sustained, so both accepted K and V need q8 and the draft + KV can stay q4. The updated default 35B guard then passed at `103.3 tok/s` + short and `90.1 tok/s` sustained. + Sampler-chain trims and backend sampling did not beat the promoted sustained + profile: `top_k;top_p;temperature` reached `104.2` / `87.5`, the milder + `penalties;top_k;top_p;min_p;temperature` chain reached `104.1` / `88.9`, + and `--backend-sampling` reached `104.3` / `89.2`, so the default sampler + path stays promoted. + A single-sequence MTP `process()` fast path for `-np 1` was also prototyped, + built, and rejected because the guard dropped to `104.1 tok/s` short and + `88.5 tok/s` sustained; the change was removed. + MoE `rows_per_block` compile-time variants were checked against the same 35B + guard and also rejected: `rows_per_block=4` measured `103.8` short / + `89.1 tok/s` sustained, `rows_per_block=3` measured `86.8 tok/s` sustained, + and `rows_per_block=1` measured `103.6` / `88.7 tok/s`, none beating the + promoted `104.3` / `89.3` band. + The Pi server profile was also started and stopped successfully with + `n_ctx = 262144`, `draft-mtp` initialized, built-in tools enabled, and + `thinking = 1`; ROCm reported no KFD PIDs after shutdown. The promoted + profile is now covered by the focused + `scripts/check-rocmfp4-qwen35-a3b-mtp-regression.sh` guard, which defaults + to `n-max 3`, q8 main KV, q4 draft KV, and `p-min 0.25` with `100.0` short + and `85.0 tok/s` sustained floors. + Follow-up `p-min 0.25/0.50/0.75`, `p-split 0.05/0.20`, and `n-min 1` + checks on the q4-KV baseline all tied the same `80.5`-`80.8 tok/s` sustained + band, while `p-min 0.25` on the q8-main/q4-draft path tied at `85.4 tok/s` + sustained, so the + conservative `n-min 0`, `p-min 0.0`, `p-split 0.10` defaults remained + promoted until the MTP internal sampler was changed from `top_k=1` to + `top_k=10`, making the `p-min` cutoff operate on a meaningful candidate + distribution while the draft loop still selects the top sorted candidate. + A pre-change p-min sweep on the actual promoted `n-max 3`, q8-main/q4-draft + profile measured `104.2` / `89.1` at `p-min 0.25`, `104.1` / `89.0` at + `0.50`, `104.0` / `89.3` at `0.75`, and `104.2` / `89.1` at `0.90`. + Follow-up p-split checks measured `103.7` / `88.9` at `0.05` and + `103.8` / `89.3` at `0.20`; n-min checks measured `103.8` / `88.9` for + `1` and `104.2` / `89.2` for `2`. None beat the promoted + sustained-plus-short profile. Combined `p-min`/`p-split` follow-ups also + tied rather than beat the default: `p-min 0.25` with `p-split 0.30` + repeated at `104.6` / `89.5`, `p-split 0.40` and `0.50` measured + `104.6` / `89.5` and `104.4` / `89.5`, and `p-split 0.70` / `0.90` + measured `104.8` / `89.5` and `104.7` / `89.5`; the same-session default + tied at `104.7` / `89.5`. With `top_k=10`, the 35B A3B profile now promotes + `--spec-draft-p-min 0.25`: default `p-min 0.0` measured `104.3` / `89.3`, + while `p-min 0.25` measured `103.9` / `90.0` after a first sustained run at + `89.8 tok/s`. The dense 27B profile should stay at `p-min 0.0`; the same + `p-min 0.25` filter regressed it to `24.6 tok/s` sustained. The full + all-regression harness can include the 35B guard with + `INCLUDE_QWEN35_A3B_GUARD=1`. + The MTP draft sampler now uses a probability-only top-10 helper for this + path. The draft loop only consumes the sorted top candidate and its + probability, so skipping the unused final RNG sampler selection preserved the + output path while improving the 35B A3B guard to `104.6 tok/s` short and + `90.2 tok/s` sustained. The dense 27B guard remained stable at `33.9` / + `28.1 tok/s`. + A fixed insertion top-10 replacement for `std::partial_sort` was tested and + rejected after the 35B A3B short guard dropped to `73.0 tok/s`, below the + `100.0 tok/s` floor. A `std::nth_element` plus top-k slice sort variant was + also rejected after the same guard dropped to `71.5 tok/s`. + Narrowing the MTP top-k probability accumulator from `double` to `float` was + also rejected: dense 27B still passed at `34.0` / `28.1 tok/s`, but the 35B + A3B short guard repeated below floor at `84.1` then `93.4 tok/s`; restoring + the `double` accumulator recovered the short guard to `104.4 tok/s`. + A `std::partial_sort_copy` top-k buffer variant built and passed the same + guard at `104.4 tok/s` short and `90.2 tok/s` sustained, but a same-session + promoted-build comparison measured `104.1` / `90.2`, so it was not promoted: + it added sampler buffer/clone complexity without a sustained decode gain. + Jackrong Qwopus3.6 27B v2 MTP BF16 was converted to STRIX_LEAN ROCmFP4 at + `4.34 BPW`. At native `262144` context with reasoning on, q4 main/draft KV, + and `n-max 4`, ROCm0 initially measured `34.9` / `29.6 tok/s`; increasing + only the batch to `-b 1024 -ub 512` moved sustained decode to `29.9 tok/s`. + Follow-up batch shapes `1280/512`, `1536/512`, and `1536/768` measured + `29.8`, `29.9`, and `29.8 tok/s` sustained respectively, so the smaller + `1024/512` profile remains promoted. Light acceptance filters `p-min 0.05` + and `0.10` also tied at `29.9 tok/s` sustained without beating the default. + Lower draft-depth checks with the same promoted batch rejected `n-max 1`, + `2`, and `3`, which measured only `19.9`, `26.6`, and `27.3 tok/s` + sustained. Qwopus therefore stays on `n-max 4`, unlike the 35B A3B profile + where `n-max 3` plus q8 main KV is best. + `--backend-sampling` tied sustained decode at `29.9 tok/s` but lowered prompt + throughput, so it is not promoted. Thread split checks at target/draft + `12/32` and `24/32` also tied `29.9 tok/s` sustained; the simpler default + `16/32` thread shape remains the recommended Qwopus profile. + KV isolation confirmed this is not like the 35B A3B q8-main profile: + draft-only q8 KV measured `35.1` / `29.8 tok/s`, and full q8 main plus q8 + draft KV measured `36.6` / `26.0 tok/s`. The full-q8 path improves burst + only and regresses sustained decode. Split accepted-KV checks also regressed: + q8 K only measured `35.7` / `22.3 tok/s`, and q8 V only measured `34.2` / + `24.7 tok/s`. Qwopus therefore keeps q4 main and q4 draft KV. + Vulkan0 measured `40.0` / `27.7 tok/s`, with Vulkan `n-max 3` and `n-max 5` + slower sustained; Vulkan `-b 1024 -ub 512` repeated the same `27.7 tok/s` + sustained band. q8 main KV regressed this model on both ROCm0 and Vulkan0. + A normal-path shortcut that only normalized `data[0].p` and filled the rest + of the top-k probabilities only for debug logging was rejected after the + 35B A3B short guard fell to `92.7 tok/s`, below the `100.0 tok/s` floor. + Disabling internal sampler timing with `sparams.no_perf = true` was also + tested and rejected after the same short guard dropped to `96.2 tok/s`. + Skipping the per-draft `common_sampler_reset()` in the MTP path was rejected + after the 35B A3B short guard dropped to `68.3 tok/s`; that reset remains + required to preserve the expected sampler/logit state for this helper. + A small MTP host-path cleanup now delays `llama_get_embeddings_pre_norm_ith()` + until the draft loop has confirmed that another draft token will be queued. + This avoids unused embedding-row pointer fetches on p-min rejects and final + `n-max` draft tokens. It passed the dense 27B guard at `34.0` / `28.1 tok/s`, + the 35B A3B guard at `104.4` / `90.1 tok/s`, and the Qwopus best ROCm0 + profile at `35.0` / `29.8 tok/s`. The default all-regression gate also + passed after this cleanup and ended with no KFD PIDs running. The 2026-05-25 + serial pass measured Qwen3.6 27B MTP at `33.9` / `27.9 tok/s`, ROCm runtime + FAST `45.66` / `57.81` / `88.27` / `155.05` us and dual-scale `49.16` / + `51.58` / `83.34` / `151.42` us for `n=1/2/4/8`, ROCm FlashAttention + `70.86` / `66.51` us for 64d dual-scale / FAST and `189.45` / `172.73` us + for Qwen-style 128d dual-scale / FAST, and ROCm CPY source-to-dual + `1106.89` / `1008.56` / `1006.60` us with source-to-FAST `1050.49` / + `958.98` / `950.50` us for F32/F16/BF16. + A second attempt to skip the final `n-max` `common_sampler_accept()` call was + rejected: the 35B A3B short check still reached `104.5 tok/s`, but sustained + decode fell to `81.1 tok/s`, below the `85.0 tok/s` floor. Reverting only + that sampler-accept change recovered the guard to `104.3` / `90.0 tok/s`. + Retesting the internal MTP sampler candidate count on the 35B A3B + reasoning-on profile rejected both directions around the promoted `top_k=10` + setting: `top_k=5` fell to `77.3 tok/s` sustained and `top_k=20` fell to + `69.6 tok/s` sustained. +- Reasoning-off checks on the final 35B q8-main/q4-draft profile are a + separate lower-throughput mode. `n-max 1/2/3/4` measured `77.7` / `73.9`, + `90.3` / `75.5`, `100.3` / `71.9`, and `85.7` / `66.1` short/sustained + tok/s respectively. If reasoning is disabled, `n-max 2` is currently the + best sustained profile in this bracket; the promoted fastest profile remains + reasoning-on `n-max 3`. +- Draft-thread-only checks on the promoted 35B profile also tied below the + promoted band. Keeping target threads at `16/32`, draft `8/16`, `16/16`, + and `24/32` all measured around `104.1`-`104.2 tok/s` short and + `89.1 tok/s` sustained, so the default matching draft thread counts remain. +- ROCm/HIP single-token MMVQ uses a full-block vector-dot ratio for the + dual-scale layout (`VDR_ROCMFP4_Q8_1_MMVQ=4`) while keeping the FAST layout + on the previous half-block ratio (`VDR_ROCMFP4_FAST_Q8_1_MMVQ=2`). This + lets dual-scale ROCmFP4 consume one full 32-value block per vector-dot call + without slowing the FAST dense-tensor path used by STRIX_LEAN. The focused + ROCm0 `MUL_MAT` guard improved dual-scale from the prior `78.81` us/run + serial pass to `54.89` us/run, and the Qwen3.6 27B MTP sustained guard + improved from `26.2` to `27.8 tok/s`. A broader version that also moved + FAST to `vdr=4` was rejected because sustained Qwen MTP dropped to + `24.2 tok/s`. A later FAST-only retest after the packed-byte load improved + the focused FAST ROCm guard to `41.37`, `49.29`, `80.91`, and `139.58` + us for `n=1/2/4/8`, but was still rejected because sustained Qwen MTP + dropped to `24.7 tok/s`. The remaining narrower FAST setting, + `GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR=1`, was also rejected after it failed the + focused FAST ROCm `n=1` guard at `60.18` us/run. The knob now rejects + invalid FAST MMVQ values at compile time; only `1`, `2`, and `4` are valid. +- A direct ROCmFP4 `vec_dot_q_cuda_dispatch` wrapper was tested in the + MMVQ kernels to bypass the generic constexpr function pointer call. It built + and passed the focused ROCm guard, measuring FAST `45.16` / `57.52` / + `89.44` / `156.49` us and dual-scale `50.83` / `51.11` / `84.51` / + `143.27` us for `n=1/2/4/8`, but Qwen3.6 27B MTP only reached `33.7 tok/s` + short and `27.9 tok/s` sustained. Because it did not beat the promoted + sustained band, the code change was removed. +- ROCm/HIP batched MMQ keeps the upstream-style `vdr=8` default for both + ROCmFP4 layouts, with ROCmFP4-owned compile-time test knobs + `GGML_ROCMFP4_Q8_1_MMQ_VDR` and `GGML_ROCMFP4_FAST_Q8_1_MMQ_VDR`. + FAST-only `vdr=4`, FAST-only `vdr=16`, and dual-scale `vdr=16` were tested + against the Qwen3.6 27B STRIX_LEAN ROCmFP4 bench and did not improve decode; + the tested runs all stayed at `13.56 tok/s` generation, while dual-scale + `vdr=16` slightly reduced prompt throughput. The default remains `8`. +- ROCm/HIP RDNA3.5 small-batch MMVQ now uses a ROCmFP4-specific two-warp + launch geometry for the ROCmFP4 layouts through `n=2`. Strix Halo previously + inherited the older RDNA2 one-warp table for this path. The promoted + `GGML_ROCMFP4_RDNA35_NWARPS=2` and + `GGML_ROCMFP4_RDNA35_NWARPS_MAX_NCOLS=2` defaults keep the single-token + microbench win and improve the guarded ROCm0 `n=2` shape: the latest serial + pass measured FAST/dual `66.56` / `58.40` us/run for `n=2`, versus the prior + `68.85` / `60.98` us/run band before extending the two-warp route. The + Qwen3.6 27B MTP guard held `33.5 tok/s` short and `27.7 tok/s` sustained, so + this is promoted as a backend micro-optimization with no sustained decode + regression. `GGML_ROCMFP4_RDNA35_NWARPS=4` was rejected because dual-scale + regressed to `57.87` us/run. The remaining 8-warp candidate was also rejected + because FAST `n=1` regressed to `59.33` us/run and failed the focused ROCm + runtime guard before multi-column checks. Extending the promoted two-warp + launch from `n=1` to `n=1..4` improved some focused multi-column ROCm + microbench rows (`dual n=2` reached `57.14` us/run), but Qwen3.6 27B MTP + sustained decode fell to `23.6 tok/s`, so the promoted upper bound stops at + `n=2`. A midpoint `GGML_ROCMFP4_RDNA35_NWARPS_MAX_NCOLS=3` build was also + checked on the Qwen3.6 35B A3B reasoning-on 262k profile; it reached + `87.5 tok/s` sustained versus `89.6 tok/s` for the same-session promoted + build, so `n=2` remains the default. +- ROCm/HIP RDNA3.5 wide-column rows-per-block now has ROCmFP4-owned compile-time + test knobs: `GGML_ROCMFP4_RDNA35_RPB_WIDE`, + `GGML_ROCMFP4_RDNA35_RPB_WIDE_DUAL`, and + `GGML_ROCMFP4_RDNA35_RPB_WIDE_FAST`. The defaults stay at `1`. A full + `RPB_WIDE=2` test improved FAST `n=8` `MUL_MAT` from `167.90` to + `131.72` us/run, but dual-scale collapsed from `148.11` to `1382.72` + us/run. A FAST-only `RPB_WIDE_FAST=2` build kept dual-scale safe + (`145.75` us/run) and improved FAST to `135.31` us/run, but Qwen3.6 + 27B MTP did not improve (`33.3 tok/s` short and `27.6 tok/s` sustained + at `n-max 4`; `n-max 5` remained burst-only at `45.5` short and `24.9` + sustained). Keep the knob off by default until a real decode guard benefits. +- ROCm/HIP RDNA3.5 `MUL_MAT_ID` routing now has a ROCmFP4-only compile-time + guard knob, `GGML_ROCMFP4_RDNA35_MMID_MAX_BATCH`, for testing whether MTP/MoE + batches should leave MMVQ earlier. The default keeps the accepted generic + RDNA3 behavior (`MMVQ_MAX_BATCH_SIZE`). A Strix test with the knob set to `1` + tied the promoted Qwen3.6 27B MTP path at `33.4 tok/s` short and `27.7 tok/s` + sustained. Follow-up threshold `2`, `3`, and `4` builds also tied at + `33.6` / `33.8` / `33.7 tok/s` short and `27.7` / `27.9` / `27.9 tok/s` + sustained. The same `3` threshold was then checked on the MoE-heavy + Qwen3.6 35B A3B MTP ROCmFP4 path and regressed to `95.8 tok/s` short and + `74.0 tok/s` sustained versus the promoted `104.4` / `89.3` band. Reusing + the older threshold `4` exploratory build on the same 35B guard measured + `104.2` / `89.2`, a near tie but still below the sustained promoted band. + No lower routing threshold is promoted. +- Target/draft "dual-stream" MTP overlap was inspected and is not a promoted + optimization. The current common MTP implementation verifies with the target + context, mirrors pre-norm embeddings into the MTP context, then drafts through + serial `llama_decode(ctx_dft, ...)` calls. ggml exposes async graph execution + and pipeline parallelism, but llama enables pipeline parallelism only for + multi-device layer-split offload cases. On single-device Strix Halo ROCm0, + target/draft overlap would require speculative scheduler changes and new + correctness guards rather than a runtime flag. The long-context optimization + focus remains KV-cache traffic, FlashAttention, memory bandwidth, and MTP + acceptance, while dual-scale ROCmFP4 mainly protects coherence. +- The MTP host loop now avoids two small sources of scheduler overhead. It + reuses a per-sequence drafting-state buffer instead of allocating a + `std::vector` on every draft call, and it tracks verify-batch sequence + bounds with one pass over token seq-ids instead of scanning every sequence for + every token. It also avoids copying all target verify rows when no previous + draft is pending, keeping only the carryover hidden row needed for the next + MTP step. The MTP path also uses a single-sequence batch append helper + instead of constructing a temporary sequence vector for each `common_batch_add` + call. It stores only non-final verify rows for partial-accept rollback, + copying the final row directly to `pending_h` for the next MTP step. It also + skips the draft-model decode call entirely when no sequence is drafting. + Debug candidate token formatting is skipped unless debug logging is enabled, + and the debug verbosity state is hoisted once per draft call instead of + rechecked inside the per-token loop. This is mainly a cleanup for + multi-sequence and long-running MTP sessions; the latest single-sequence + Qwen3.6 27B guard stayed in range at `33.7 tok/s` short and `27.9 tok/s` + sustained. +- The simple draft path also reuses an object-owned `uint8_t` drafting-state + buffer instead of allocating `std::vector` on every draft call. This + keeps both draft implementations on the same host-side allocation pattern; + the full serial gate stayed clean, with Qwen3.6 27B MTP at `33.8 tok/s` + short and `27.9 tok/s` sustained. +- The all-in-one regression harness now accepts a candidate `BUILD_DIR` and + derives separate `TEST_BACKEND_OPS_BIN` and `LLAMA_CLI_BIN` paths, so backend + microbench guards and Qwen CLI guards can run from the same candidate build. + DeepSeek remains opt-in only through `INCLUDE_DEEPSEEK_SMOKE=1`. +- ROCm/HIP vector FlashAttention now uses a ROCmFP4-only RDNA K/Q thread-group + default of `1` instead of the generic quantized-K RDNA default of `2`. This + affects only `Q4_0_ROCMFP4` and `Q4_0_ROCMFP4_FAST` K-cache cases; other + quantized FA paths keep the existing backend default. On the guarded ROCm0 + `FLASH_ATTN_EXT` shape, dual-scale improved from `122.33` to `113.03` + us/run, and FAST improved from `115.41` to `109.33` us/run. The Qwen3.6 + 27B MTP guard remained in the promoted band at `33.5 tok/s` short and + `27.6 tok/s` sustained. A wider `GGML_ROCMFP4_FATTN_KQ_NTHREADS=4` variant + was rejected because it regressed the same FA guard to `136.79` us/run + dual-scale and `124.37` us/run FAST. After the V-side default moved to `2`, + retesting `KQ_NTHREADS=2,V_NTHREADS=2` and `KQ_NTHREADS=4,V_NTHREADS=2` + still regressed the focused FA guard, so K/Q remains `1`. +- ROCm/HIP vector FlashAttention also uses a ROCmFP4-only V thread-group + default of `2`, down from the generic `D/4` path used by other quantized V + types. On the same guarded ROCm0 `FLASH_ATTN_EXT` shape, this moved + dual-scale from `113.03` to `85.63` us/run and FAST from `109.33` to + `80.74` us/run. The Qwen3.6 27B MTP guard stayed in range at `33.4 tok/s` + short and `27.6 tok/s` sustained, so this is promoted as the new default. + `GGML_ROCMFP4_FATTN_V_NTHREADS=4` also passed but was slower on the focused + FA guard; `1` was rejected because gfx1151 HIP compilation exceeded the 64 + KiB local-memory limit for ROCmFP4 FA instances. The FA guard now also + includes a Qwen3.6-style 128-head-dim, 8-KV-head, 12x-GQA ROCmFP4 shape + (`hsk=128,hsv=128,nh=8,nr23=[12,1],kv=7680,nb=1`). The accepted default + measured `246.91` us dual-scale and `219.23` us FAST on that shape. + Retesting `V_NTHREADS=4` on this wider shape produced a tiny dual-scale + improvement (`244.01` us) but regressed FAST (`224.21` us) and the existing + 64d guard, so it remains rejected. Retesting `KQ_NTHREADS=2,V_NTHREADS=2` + regressed the Qwen-style shape to `268.53` us dual-scale and `226.32` us + FAST, so K/Q remains `1`. +- ROCm/HIP vector FlashAttention now uses a ROCmFP4-owned single-half + Codebook10 expander in K/Q and V decode paths. These FA call sites already + know whether they need the low or high nibble stream, so they no longer pay + to expand both streams and discard one. The focused ROCm0 FA guard improved + from `86.13` / `81.23` us to `82.24` / `78.15` us for the 64d dual-scale / + FAST shapes, and from `247.06` / `221.19` us to `237.12` / `206.50` us for + the Qwen-style 128d shapes. The Qwen3.6 27B MTP guard held at `33.5 tok/s` + short and `27.7 tok/s` sustained in the full serial gate, so this is + promoted. +- ROCm/HIP vector FlashAttention also specializes the ROCmFP4 K/Q path when + `GGML_ROCMFP4_FATTN_KQ_NTHREADS=1`. Each thread owns the full head dot in + this promoted setting, so the kernel now expands both low/high packed + Codebook10 streams once per ROCmFP4 block and accumulates both half-blocks + together instead of loading the same packed bytes twice. The full serial gate + measured 64d FlashAttention at `81.62` / `78.13` us for dual-scale / FAST, + and the wider Qwen-style 128d guard improved from `237.12` / `206.50` us to + `228.58` / `199.32` us. Qwen3.6 27B MTP held `33.2 tok/s` short and + `27.7 tok/s` sustained, so this is promoted as a real Qwen-relevant FA + micro-optimization. + A post-specialization retest of `GGML_ROCMFP4_FATTN_V_NTHREADS=4` was + rejected. It passed the tightened FA guard and nudged the Qwen-style + dual-scale microbench to `228.33` us, but regressed 64d dual-scale to + `86.30` us and Qwen-style FAST to `201.81` us. Qwen3.6 27B MTP stayed at + `27.7 tok/s` sustained, so the promoted V-side default remains `2`. +- ROCm/HIP vector FlashAttention now uses a ROCmFP4-only V dequant rows-per- + thread default of `8`. The ROCmFP4 V helper supports 8-value chunks, and the + post-unaligned-load retest produced a real FA win: 64d dual-scale / FAST + `68.82` / `66.32` us and Qwen-style 128d dual-scale / FAST `201.13` / + `172.22` us. Qwen3.6 27B MTP held `33.5 tok/s` short and `27.7 tok/s` + sustained, so this is now promoted. It can be isolated with + `-DGGML_ROCMFP4_FATTN_V_ROWS_PER_THREAD=4` to return to the prior setting. + Retesting `GGML_ROCMFP4_FATTN_V_NTHREADS=4` on top of the 8-row default + improved only the Qwen-style dual-scale FA microbench (`187.72` us) while + regressing 64d dual-scale / FAST to `77.35` / `77.25` us and Qwen-style FAST + to `182.56` us. Qwen3.6 27B MTP tied at `33.5` / `27.7 tok/s`, so the + shipped V thread-group default remains `2`. + Retesting `GGML_ROCMFP4_FATTN_V_NTHREADS=8` on top of the 8-row default + regressed every focused FA row, including Qwen-style 128d dual-scale / FAST + at `208.94` / `200.45` us, so it was rejected without a longer MTP run. + Retesting `GGML_ROCMFP4_FATTN_KQ_NTHREADS=2` on top of the 8-row default + also regressed the focused FA guard: 64d dual-scale / FAST measured + `80.68` / `78.27` us and Qwen-style 128d measured `245.84` / `207.04` us. + Because the Qwen-style dual-scale row failed the guard, it was rejected + without a longer MTP run and K/Q remains `1`. +- ROCm/HIP vector FlashAttention now has a narrow dual-scale 128d V-thread + specialization: `GGML_ROCMFP4_FATTN_V_NTHREADS_D128_DUAL=4`. This keeps the + promoted 64d default and all FAST paths on `V_NTHREADS=2`, but uses the + previously promising 4-thread V grouping only for the Qwen-style dual-scale + 128d shape. The full serial gate measured FA at `70.37` / `66.31` us for + 64d dual-scale / FAST and `188.75` / `171.89` us for Qwen-style 128d + dual-scale / FAST, while Qwen3.6 27B MTP held `33.7 tok/s` short and + `27.9 tok/s` sustained with no KFD PIDs left running. + Retesting `GGML_ROCMFP4_FATTN_V_NTHREADS_D128_DUAL=8` was rejected. It + slightly improved the 64d dual-scale row (`69.40` us versus a same-session + `70.92` us promoted build), but regressed the Qwen-style 128d dual-scale row + from `194.40` to `211.10` us/run. Because long-context Qwen is the relevant + guard, the promoted D128 dual-scale V grouping remains `4`. + `GGML_ROCMFP4_FATTN_V_ROWS_PER_THREAD=16` was rejected during compilation: + the ROCmFP4 V dequantizer intentionally supports only `2`, `4`, and `8` + rows per thread, and the fixed-copy helper does not support the resulting + 32-byte move. The source now rejects unsupported values with a direct + ROCmFP4 compile-time error. + Retesting `GGML_ROCMFP4_FATTN_V_ROWS_PER_THREAD=2` on the promoted + D128-specialized build was rejected by the focused ROCm FlashAttention + guard. It regressed 64d dual-scale / FAST to `94.31` / `88.77` us and + Qwen-style 128d dual-scale / FAST to `270.00` / `223.88` us, so the + promoted rows-per-thread default remains `8`. +- Vulkan ROCmFP4 scale decode now uses a shared UE4M3 lookup table with + ROCmFP4's half-scale semantics. This moved the focused Vulkan dual-scale + `MUL_MAT` guard from `82.86`, `120.77`, and `181.28` us/run to `65.05`, + `83.07`, and `122.70` us/run for `n=1/2/4`. On Qwen3.6 27B MTP at 262k + context, the same change moved Vulkan sustained output from the older + `20.4 tok/s` at `--spec-draft-n-max 4` to `25.0 tok/s`; `n-max 3` now + reaches `25.3 tok/s` sustained. ROCm0 remains the promoted backend because + it still holds `33.5 tok/s` short and `27.7 tok/s` sustained. + A post-LUT Vulkan runtime sweep found a better fallback profile with q8 KV + and q8 draft KV: `--spec-draft-n-max 4` reached `34.8 tok/s` short and + `27.0 tok/s` sustained. q8 with `n-max 5` improved only the short burst + (`47.8 tok/s`) and regressed sustained output to `23.0 tok/s`; f16 KV + regressed sustained output to `22.5 tok/s`. + Follow-up isolation showed the Vulkan sustained gain comes from the main KV + cache rather than the draft KV cache: q4 main KV with q8 draft KV reached + only `34.6 tok/s` short and `25.0 tok/s` sustained, while q8 main KV with + q4 draft KV reached `34.7 tok/s` short and `26.9 tok/s` sustained. For a + Vulkan fallback, q8 main KV plus q4 draft KV is therefore the leaner near-tie + profile. Splitting q8 across only one accepted KV side was also rejected: + q8 main K with q4 main V reached `34.8 tok/s` short and `25.4 tok/s` + sustained, while q4 main K with q8 main V reached `34.6 tok/s` short and + `23.7 tok/s` sustained. The Vulkan fallback needs both accepted K and V at + q8. Adding mild acceptance filtering to the lean fallback + (`--spec-draft-p-min 0.25`) tied full q8/q8 at `27.0 tok/s` sustained while + keeping draft KV at q4; adding `--spec-draft-n-min 1`, stricter `p-min 0.75`, + and `p-split` checks stayed at `26.9 tok/s`. Retrying `n-max 5` with q8 main + KV, q4 draft KV, and + `p-min 0.25` still produced only a burst win (`47.8 tok/s` short) while + sustained output stayed at `23.0 tok/s`, so `n-max 4` remains the Vulkan + fallback setting. ROCm0 q4/q4 remains the overall promoted backend/profile. + Replacing the shared Codebook10 table with inline integer decode was tested + and rejected because it regressed the focused Vulkan FAST `n=1` guard to + `99.43` us/run. +- ROCm/HIP fallback dequant, copy, get-rows, GPU-side quantization scoring, + and standalone dequant helpers use ROCmFP4-owned HIP helpers for finite + scales and Codebook10 nibbles instead of relying on generic FP8 handling. + This keeps non-MMQ conversion paths aligned with the custom AMD format. +- A branchless HIP scalar Codebook10 nibble decoder was tested and rejected. + It passed the focused ROCm CPY, FlashAttention, and Qwen MTP guards, but + did not improve end-to-end decode. Qwen3.6 27B MTP measured `33.4 tok/s` + short and `27.7 tok/s` sustained, and the focused FlashAttention guard + slowed to `86.38` / `81.13` us for dual-scale / FAST, so the original + scalar decoder remains the default. +- CPU-side Codebook10 table decode in the quantizer/dequantizer was tested + and rejected. The full table variant failed the CPU quant guard with + dequantization at `49.04` cycles/32 for dual-scale and `84.23` cycles/32 for + FAST. The MSE-loop-only variant passed but slowed normal quantization to + `4183.33` / `4018.68` cycles/32 versus the restored arithmetic baseline at + `4034.33` / `3738.85` cycles/32, so the arithmetic CPU decoder remains the + default. +- UE4M3 scale decode in the ROCm/HIP software path uses a ROCmFP4-owned finite + scale decoder. It avoids `ldexpf`, builds normal FP32 values directly from + exponent/mantissa bits, and skips the generic FP8 NaN handling because + ROCmFP4 row validation already rejects non-finite scale bytes. +- A constant-memory HIP lookup table for the finite UE4M3 scale values was + tested and rejected. The latest isolated `GGML_ROCMFP4_USE_SCALE_LUT=1` + pass failed the focused ROCm runtime guard because FAST `n=1` regressed to + `69.48` us/run; prior checks also regressed ROCm CPY source-to-quant paths + and FlashAttention. The arithmetic finite-scale decoder remains the default. +- ROCm/HIP dequant conversion kernels use the same ROCmFP4 finite scale + decoder, keeping tensor conversion aligned with the hot MMQ/MMVQ backend + path instead of falling back to the generic FP8 helper. +- ROCm/HIP backend CPY now advertises and executes quantized + `Q4_0_ROCMFP4 -> F32` and `Q4_0_ROCMFP4_FAST -> F32` conversion paths. + This keeps diagnostic graph ops and fallback tensor conversion inside the + custom AMD decoder instead of being rejected by backend capability checks. + Contiguous q-to-f32 copies now use a ROCmFP4-specific packed-byte kernel: + one HIP thread reads one packed byte and writes the matching low/high + half-block output values. This gives coalesced output writes for normal + contiguous graph copies while the existing stride-aware block kernel remains + the fallback for views and non-contiguous tensors. +- ROCm/HIP backend CPY also supports `F16 -> Q4_0_ROCMFP4`, + `F16 -> Q4_0_ROCMFP4_FAST`, `BF16 -> Q4_0_ROCMFP4`, and + `BF16 -> Q4_0_ROCMFP4_FAST`. The kernels convert each 32-value half/bfloat + block to local FP32 and then run the same exhaustive ROCmFP4 scale search, + so runtime graph copies keep the coherence-first quantizer instead of + falling back to unsupported behavior. +- ROCm/HIP ROCmFP4 source-to-quant and quant-to-F32 CPY wrappers now use + normal multi-thread HIP launch geometry instead of launching one active + thread per quant block. This keeps F32/F16/BF16 runtime quantization and + ROCmFP4 dequantization on the same exact conversion math while removing a + launch-shape bottleneck in helper/fallback graph paths. +- ROCm/HIP ROCmFP4 quant-to-F32 CPY uses ROCmFP4-specific block dequant + helpers that decode each block scale once before unpacking all 32 values. + This avoids repeatedly decoding identical scale bytes through the generic + two-value dequant helper while preserving the same output values. For the + guarded contiguous `8192x512x2` ROCm0 CPY shape, the packed-byte path moved + dual-scale `Q4_0_ROCMFP4 -> F32` from the old `740` us/run band to + `184.48` us/run and FAST `Q4_0_ROCMFP4_FAST -> F32` to `169.99` us/run. + A split launch geometry is now used: source-to-ROCmFP4 quantization uses + 128-thread workgroups, while quant-to-F32 keeps the accepted 64-thread + packed-byte launch. A whole-path 128-thread launch was roughly tied overall + but slightly regressed quant-to-F32; isolating 128 threads to source + quantization preserves the dequant copy win while shaving the source paths. + A 256-thread launch regressed F16 source-to-quant paths and remains rejected. + After the FAST direct-value scoring win, FAST-only 256-thread and 64-thread + source launch splits were rechecked and rejected as well. The 256-thread + split regressed FAST F16 source-to-quant to `1047.24` us/run, and the + 64-thread split measured FAST F32/F16/BF16 `1055.32`, `955.35`, and + `954.78` us/run, so the shared 128-thread source launch remains promoted for + both ROCmFP4 layouts. +- ROCm/HIP contiguous quant-to-F32 CPY shared-scale staging was tested and + rejected. Decoding one block scale into shared memory and synchronizing each + 64-thread launch reduced duplicate scale decode work, but the synchronization + overhead regressed the guarded `8192x512x2` shape: dual-scale + `Q4_0_ROCMFP4 -> F32` moved from `181.84` to `188.63` us/run, and FAST + `Q4_0_ROCMFP4_FAST -> F32` moved from `170.65` to `180.03` us/run. The + direct packed-byte kernel remains promoted. +- ROCm/HIP backend CPY supports same-type packed-block copies for + `Q4_0_ROCMFP4 -> Q4_0_ROCMFP4` and + `Q4_0_ROCMFP4_FAST -> Q4_0_ROCMFP4_FAST`, including block-aligned views. + The kernel copies the packed 18-byte or 17-byte ROCmFP4 blocks directly, so + graph/view copies preserve exact bytes and avoid dequantize/requantize + fallback behavior. The launcher uses normal multi-thread HIP workgroups + rather than one-thread launches, so large packed-view copies scale with the + number of ROCmFP4 blocks. +- ROCm/HIP runtime quantization now finds the finite UE4M3 candidate nearest + `max_abs / 10` with a monotonic binary search, matching the CPU/Vulkan + reference tie behavior while avoiding the older 126-step linear nearest-scale + scan in the HIP copy utility. It also uses the same conservative lower-scale + pruning as the CPU quantizer, so runtime F32/F16/BF16-to-ROCmFP4 copies skip + smaller scales once their unavoidable clipped max-value error cannot beat the + current best exact scale. The HIP max scan uses the same plain absolute + compare as the CPU quantizer, preserving NaN/Inf handling while avoiding + `fmaxf` in the runtime quantization hot loop. +- ROCm/HIP runtime quantization specializes the exact scale search for the + only block shapes used by ROCmFP4 CPY: `0..15`, `16..31`, and `0..31`. + This keeps the same candidate order and tie behavior while letting HIP + compile the dual-scale and FAST source-to-quant paths with fixed offsets. +- ROCm/HIP dual-scale source-to-quant scoring uses a direct Codebook10 value + helper for 16-value half blocks. Final packed quantization still uses the + normal code-index helper, but scale scoring no longer has to map the selected + nibble back through the generic decode helper on the dual-scale path. +- ROCm/HIP FAST source-to-quant scoring now uses the same direct Codebook10 + value helper during scale search. This preserves the exact nearest-value + thresholds and final packed output, but avoids index-then-decode work in the + 32-value FAST scoring path. On the guarded `8192x512x2` ROCm0 CPY shape, + FAST source quantization improved from F32/F16/BF16 `1218.69`, `1138.32`, + and `1138.78` us/run to a repeat guard pass of `1047.21`, `950.93`, and + `951.00` us/run. +- ROCm/HIP `F32 -> ROCmFP4` runtime CPY stages each 32-value block into local + FP32 before running the exhaustive scale search, matching the existing + F16/BF16 source paths. This preserves exact F32 input values but avoids + rereading the global source pointer throughout scale scoring. On the guarded + `8192x512x2` ROCm0 CPY shape, dual-scale F32 source quantization improved + from roughly `9916` to `1117` us/run, and FAST improved from roughly + `10671` to `1231` us/run. +- ROCm/HIP contiguous-only source-to-quant CPY kernels for + `F32 -> ROCmFP4` were tested and rejected. They avoided the generic + multidimensional offset math, but on the guarded `8192x512x2` shape they + did not beat the existing multi-thread generic CPY path and slightly + regressed `F32 -> Q4_0_ROCMFP4_FAST`, so source-to-quant ROCmFP4 CPY keeps + the same guarded implementation for view and contiguous tensors. +- ROCm/HIP backend GET_ROWS supports both ROCmFP4 layouts. This gives pure + ROCmFP4 tensors the same direct row-gather coverage as stock small-block + quants on ROCm and keeps embedding-row access on the custom finite-scale + decoder. +- ROCm/HIP `MUL_MAT` support now covers `F16` activation tensors for both + ROCmFP4 layouts. The backend stages half activations to contiguous FP32 on + the GPU, including non-contiguous/views, then feeds the existing Q8 + activation quantizer and ROCmFP4 MMVQ/MMQ kernels. This keeps the forward + path on the AMD backend instead of rejecting the op and falling through to a + slower dequantized matrix path. The generic matmul runtime guard explicitly + allows this ROCmFP4 x F16 forward-inference case, so the support probe and + execution wrapper agree for batched activations. +- The standalone HIP dequant skeleton covers both the dual-scale and FAST + single-scale layouts, so future fused ROCm kernels can target the current + balanced FAST artifact without reintroducing the older scale path. +- The standalone HIP dequant launch now maps one thread to one packed + ROCmFP4 byte across a 256-thread global grid. This avoids the older + one-16-thread-block-per-32-values launch shape and gives future fused or + diagnostic dequant paths normal GPU occupancy scaling. +- Vulkan ROCmFP4 shaders also decode UE4M3 scales directly to the half-scale + value used by the codebook, matching CPU/HIP and avoiding repeated `* 0.5` + fixups at dequant and matmul call sites. +- Vulkan ROCmFP4 shaders keep a shared `kvalues_rocmfp4` Codebook10 table. + Arithmetic/direct Codebook10 decode variants compiled and preserved + coherence, but measured slower on Strix Halo Vulkan, so the table path + remains the active backend implementation. +- Vulkan `dequantize4()` for ROCmFP4 and ROCmFP4_FAST decodes each block's + UE4M3 scale once per 4-value vector instead of calling the 2-value helper + twice. This improves tested dual-scale ROCmFP4 small-batch matvec shapes + while keeping the promoted Vulkan runtime guard clean. +- Vulkan `Q4_0_ROCMFP4_FAST` matvec/MMQ kernels have a single-scale dot + specialization. They combine the two half-block dot sums and apply the one + FAST scale once, instead of taking the dual-scale path used by + `Q4_0_ROCMFP4`. +- Vulkan `Q4_0_ROCMFP4_FAST` MMQ stores its block scale as a scalar in + shared/register cache instead of duplicating it into a `vec2`. The focused + Strix Halo Vulkan `MUL_MAT` microbench for + `m=4096,n=1,k=14336,type_a=q4_0_rocmfp4_fast` improved from `62.18` to + `61.45` us/run. +- A Vulkan packed16 view for ROCmFP4 quant bytes was tested and rejected. It + compiled, but the focused FAST MMQ guard regressed to `83.03` us/run from the + accepted `61`-`62` us/run range, so ROCmFP4 Vulkan keeps the byte-view load + path. +- Vulkan backend CPY/SET_ROWS now has generated `F32 -> Q4_0_ROCMFP4`, + `F32 -> Q4_0_ROCMFP4_FAST`, `Q4_0_ROCMFP4 -> F32`, + `Q4_0_ROCMFP4_FAST -> F32`, and indexed SET_ROWS shaders. The SET_ROWS path + uses the same exhaustive finite UE4M3 scale search as the CPU reference, so + quantized K/V cache writes favor coherence over a cheap max-abs shortcut. +- Vulkan `F32 -> ROCmFP4` runtime quantization now uses the same exact ordered + UE4M3 scale search as CPU/HIP: find the scale nearest `max_abs / 10`, expand + outward, and stop a candidate once its partial error cannot beat the current + best scale. The candidate set is unchanged, so this avoids a slower linear + scan without falling back to a lower-quality shortcut. +- Vulkan source-to-ROCmFP4 runtime quantization now also prunes lower scale + candidates once clipping the block max alone cannot beat the current best + error. This mirrors the CPU/HIP scale search bound and keeps the candidate + set exact. The 2026-05-25 Vulkan CPY guard passed after this shader change: + F32/F16/BF16-to-dual measured `9525.39`, `2350.54`, and `2418.09` us/run, + dual-to-F32 `516.65` us/run, F32/F16/BF16-to-FAST `10111.85`, `2923.67`, + and `2949.42` us/run, and FAST-to-F32 `509.65` us/run. The full promoted + gate passed after rebuild with Qwen3.6 27B MTP at `33.9` / `28.0 tok/s` and + no KFD PIDs left running. +- Vulkan backend CPY also supports `F16 -> Q4_0_ROCMFP4`, + `F16 -> Q4_0_ROCMFP4_FAST`, `BF16 -> Q4_0_ROCMFP4`, and + `BF16 -> Q4_0_ROCMFP4_FAST`. The runtime quantization shader can load + `float`, `float16_t`, and BF16 source bits and then runs the same exact + ordered finite UE4M3 scale search. Backend tests keep a bounded NMSE + tolerance only for the half/bfloat runtime quantization cases because those + paths are inherently lossy around source-precision tie points; the + `F32 -> ROCmFP4` checks remain strict. +- Vulkan same-type CPY supports packed-block copies for + `Q4_0_ROCMFP4 -> Q4_0_ROCMFP4` and + `Q4_0_ROCMFP4_FAST -> Q4_0_ROCMFP4_FAST`, including block-aligned + non-contiguous/permuted/view copies. The non-contiguous path uses a + byte-addressed block shader and preserves exact 18-byte or 17-byte + ROCmFP4 blocks. Contiguous same-type dual-scale and FAST copies now both use + the direct byte-copy path, avoiding the generic halfword copy route for + ROCmFP4's custom 18-byte and 17-byte block layouts. +- Vulkan CPY is now covered by a dedicated regression guard so copy-path + changes cannot silently fall back or regress outside the ROCm-only CPY gate. + The guard runs same-type and source/dequant CPY correctness before measuring + the large copy performance shape, and now streams the performance phase + through `tee` so long Vulkan runs do not look idle to the command runner. + The ceilings were tightened after the lower-scale pruning gain, so the old + slow source-to-quant path no longer passes this guard. + On Strix Halo RADV Vulkan, the large guarded shape currently measures + F32/F16/BF16-to-dual at `9525.39`, `2350.54`, and `2418.09` us/run, + dual-to-F32 at `516.65` us/run, F32/F16/BF16-to-FAST at `10111.85`, + `2923.67`, and `2949.42` us/run, and FAST-to-F32 at `509.65` us/run. +- Vulkan scalar FlashAttention can now decode ROCmFP4 and ROCmFP4_FAST K/V + cache blocks. ROCmFP4 K/V is forced to the scalar FA path because the current + custom decode is not a coopmat/native matrix-core FP4 path. +- Vulkan scalar FlashAttention can use the integer-dot MMQ K path for both + ROCmFP4 K-cache layouts. The FAST layout expands each 4-bit Codebook10 value + into packed signed int8 lanes and uses its single UE4M3 scale as the K block + multiplier. The dual-scale layout also uses packed signed int8 lanes, but + splits the dot accumulation by half-block so each 16-value half uses its own + UE4M3 scale. This keeps the quality-biased STRIX K-cache path fast without + applying one scale to a two-scale block. +- Build and runtime verification generated the Vulkan SPIR-V entries for + ROCmFP4 copy/SET_ROWS shaders, linked `libggml-vulkan.so`, and passed + Vulkan ROCmFP4 CPY plus MUL_MAT smoke tests on Strix Halo. +- Row validation rejects invalid scale bytes outside finite unsigned UE4M3 + (`0x00` through `0x7e`) so corrupted custom GGUF tensors fail early. +- Quantization keeps the exhaustive 126-scale UE4M3 search for both normal and + imatrix paths. Candidate-window scale search was tested and improved GGUF + creation speed, but it regressed the Qwen3-4B short WikiText-2 PPL on the + compact FAST path, so it was rejected for coherence. +- The exhaustive scale search now visits the UE4M3 candidate nearest the + block's `max_abs / 10` first using a monotonic binary search, expands + outward, and exits a candidate scale once its partial error cannot beat the + current best scale. This remains exact because every finite scale is still + evaluated; on the Qwen3-4B FAST_COHERENT artifact it produced a + byte-identical GGUF while cutting FAST quantization cost sharply. +- CPU scale decode now uses a 127-entry finite UE4M3 half-scale table for the + exhaustive scale search and row dequantization. The table preserves the same + exact FP32 values as the former bit-construction decoder, but avoids + rebuilding them for every scale candidate. +- CPU scale search now has separate unweighted and imatrix-weighted MSE + scoring paths. The normal path avoids per-value weight branches, while the + imatrix path precomputes the row-energy calibration weight once per + 16/32-value block instead of recomputing the same `sqrtf` term for every + candidate scale. +- CPU normal quantization scans each block once for finite values and uses a + finite-only nearest-Codebook10 helper in the exhaustive MSE loop when the + block is clean. Non-finite blocks still use the guarded helper. This + promoted conversion-speed path measured focused dual-scale / FAST normal + quantization at `3844.38` / `3582.57` cycles per 32 values in the latest + guard. +- CPU imatrix-weighted scale-MSE scoring now uses the same finite-only + nearest-Codebook10 helper when the source block is finite. This preserves the + guarded path for non-finite input while improving FAST imatrix GGUF creation: + same-session pre-candidate FAST imatrix was `5258.07` cycles / 32 values, and + two guarded candidate passes measured `4448.73` and `4447.32`. Dual imatrix + stayed in the noisy guarded band, so the claim is limited to FAST imatrix. +- The weighted and unweighted exhaustive scale loops are split as well, so + the normal quantizer does not branch on imatrix state for every finite + UE4M3 candidate. This keeps the exact candidate order and tie behavior while + improving CPU GGUF creation speed for both ROCmFP4 layouts. +- The CPU quantizers fill every packed nibble byte in each ROCmFP4 block, so + they no longer clear `qs` before packing. This is an exact store cleanup: + correctness stayed clean, `q4_0_rocmfp4` quantize improved to `5244.46` + cycles / 32 values, `q4_0_rocmfp4_fast` improved to `4668.28` cycles / 32 + values. +- CPU scale selection scans block maxima with a plain absolute-value compare + instead of `fmaxf`. NaNs still do not raise the maximum, infinities are still + rejected before scale search, and finite output is unchanged. This moved + `q4_0_rocmfp4` quantize to `5228.64` cycles / 32 values and FAST to + `4516.45` cycles / 32 values. +- CPU dequantization and quantization-error scoring use the ROCmFP4 arithmetic + Codebook10 decoder, which avoids a table fetch per unpacked nibble. CPU + fallback vec-dot keeps the 16-entry codebook table because measured Strix + Halo fallback dot speed was better with the table there. This hybrid keeps + the dequant win without regressing CPU vec-dot. +- CPU fallback vec-dot now reuses each packed ROCmFP4 byte for both low and + high nibbles before table decode. This preserves exact output, moved focused + dual-scale vec-dot from `31.86` to the `29.77`-`29.82` cycles / 32 values + band, and kept FAST in the `27.04`-`27.06` cycles / 32 values band. +- Two additional CPU decode shortcuts were rechecked on 2026-05-24 and + rejected. Replacing arithmetic Codebook10 decode with table decode in + row dequantization and scale-MSE scoring slowed CPU dequantization from the + guarded `33`-cycle band to `51`-`84` cycles / 32 values. Returning a direct + decoded value during MSE scoring instead of index-then-decode also passed + correctness but regressed normal quantization in the measured guard, so the + original arithmetic decoder remains the CPU path. A narrower full-block + weighted-MSE-only retry was also rejected because it regressed the focused + FAST imatrix timing to `5007.73` cycles / 32 values. +- A direct finite-scale table helper for CPU quantizer scale-search candidates + was also rejected. It helped dual-scale normal quantization, but repeat + guard runs regressed FAST normal quantization to `4043.60` and + `4289.66` cycles / 32 values. The guarded scale helper remains in place + because the compact STRIX_LEAN preset depends heavily on FAST tensors. +- NaN-only and branchless finite-scan variants were also rejected for default. + The NaN-only scan was noisy and the same-session `isfinite()` scan measured + better dual-scale normal quantization (`3735.41` vs `3776.53` cycles / 32 + values), while the branchless boolean-and scan regressed FAST normal + quantization to `3773.44` cycles / 32 values. +- Per-value Codebook10 quantization uses exact nearest-neighbor thresholds + instead of a 16-entry scan. The hot quantizer path uses one reciprocal scale + per candidate/block and multiplies each value by that reciprocal instead of + dividing per value. On the Qwen3-4B FAST_COHERENT check this kept PPL tied + with the accepted artifact while cutting GGUF creation time further. +- CPU exhaustive scale search now prunes lower-scale candidates after the + clipped max-value error alone cannot beat the current best error. The + imatrix path uses the positive calibration weight of the max element for the + same conservative bound. This does not change accepted values because every + skipped smaller scale has a larger unavoidable clip error. The weighted path + also combines imatrix weight preparation, max-absolute scan, and max-value + pruning-weight selection into one pre-scan, while both exhaustive loops stop + once no valid lower or upper scale candidates remain. Latest 40-iteration + quant guard run: `q4_0_rocmfp4` normal `4049.28` cycles / 32 values, FAST + normal `3721.48`, imatrix dual-scale `5251.76`, imatrix FAST `4898.75`. +- Adding `GGML_RESTRICT` to the local CPU scale-scoring helper pointers was + tested and rejected. It compiled and passed the quant guard, but did not + improve normal quantization and made the imatrix guard noisier/slower in + the measured pass, so the existing helper signatures were kept. +- Regression guards: + - `scripts/check-rocmfp4-deepseek-regression.sh` is an optional compatibility + smoke guard for a second ROCmFP4-converted model family. It is not part of + the promoted-gain gate because no reproducible DeepSeek speedup has been + established yet. The default all-regression script skips it unless + `INCLUDE_DEEPSEEK_SMOKE=1` is set. + - `scripts/check-rocmfp4-qwen-mtp-regression.sh` verifies the promoted + Qwen3.6 27B MTP ROCmFP4 262k-context `draft-mtp` path. The stable guard + uses ROCm0 and `--spec-draft-n-max 4`. On ROCm0, `n-max 4` improved the + short guard prompt from `27.7` to `33.6 tok/s` versus `n-max 3`, and the + longer forced-output prompt from `25.6` to `26.3 tok/s`; the dual-scale + MMVQ VDR tune then moved the longer prompt to `27.8 tok/s`. `n-max 5` + and `n-max 6` improved only the bursty short prompt after the MMVQ tune + but regressed sustained output to `24.8` and `24.0 tok/s`, respectively. + Follow-up `n-max 5` checks with `--spec-draft-p-min 0.75`, + `--spec-draft-p-min 0.90`, and `--spec-draft-n-min 1` still held the + short prompt at `45.0`-`45.1 tok/s` but kept sustained output at only + `24.7 tok/s`, so `n-max 4` remains promoted. + The guard now checks both short and sustained prompts so high-acceptance + best-case gains do not hide sustained-output regressions. + Additional ROCm0 checks rejected larger batch sizes, smaller ubatch, + q8/f16 KV cache, `--swa-full`, `--no-host`, `--no-op-offload`, + `--no-repack`, polling changes, lower and higher CPU thread counts, + greedy sampling, backend sampling, p-split changes, sampler chain trimming, + and combined MTP+ngram speculation because none improved the sustained + prompt. After the MMVQ VDR tune, `-b 1024 -ub 512`, `-b 2048 -ub 512`, + and `-b 512 -ub 256` still tied the promoted `27.8 tok/s` sustained + result, while `-b 1024 -ub 1024` and + `--poll-batch 1 --spec-draft-poll-batch 1` stayed at `27.7 tok/s`. + q8 and f16 KV cache dropped sustained decode to `25.4` and `26.0 tok/s`. + The MTP draft loop now honors `--spec-draft-p-min`. With that cutoff active, + `n-max 4, p-min 0.75` tied the promoted path at `33.5 tok/s` short and + `27.6 tok/s` sustained, while `n-max 5` stayed burst-only at `45.0`-`45.1` + tok/s short and `24.6`-`24.7 tok/s` sustained for `p-min 0.75` and `0.90`. + Follow-up ROCm0 KV isolation confirmed this is not a draft-cache issue: + q8 main KV with q4 draft KV measured `33.5 tok/s` short and `25.3 tok/s` + sustained, q8 main K only measured `31.5` and `21.4 tok/s`, and q8 main V + only measured `31.1` and `21.9 tok/s`. The promoted ROCm0 path therefore + keeps both accepted and draft KV at q4. + `--spec-draft-p-split 0.05` and `0.20` both tied the `27.7 tok/s` + sustained guard, `-t 24 -tb 32 --spec-draft-threads 24 + --spec-draft-threads-batch 32` tied, and trimming the sampler chain to + `top_k;top_p;temperature` regressed sustained decode to `26.6 tok/s`. + Draft-only q8 KV (`SPEC_DRAFT_TYPE_K=q8_0`, + `SPEC_DRAFT_TYPE_V=q8_0`) with main KV left at q4 tied the promoted path + at `33.6 tok/s` short and `27.7 tok/s` sustained, so the leaner q4 draft + KV default remains promoted. + The heavier Qwen3.6 27B MTP `STRIX_MTP_Q6` ROCmFP4 profile was also + checked with the same ROCm0 guard. It reached `30.1 tok/s` on the short + prompt but only `21.3 tok/s` sustained, below the guard floor, so the + Q6/quality-biased profile is not promoted for throughput. + The guard script supports `SPEC_DRAFT_N_MAX`, `SPEC_DRAFT_N_MIN`, + `SPEC_DRAFT_P_MIN`, `SPEC_DRAFT_P_SPLIT`, `THREADS`, `THREADS_BATCH`, + `SPEC_DRAFT_THREADS`, `SPEC_DRAFT_THREADS_BATCH`, `SAMPLERS`, + `BATCH_SIZE`, `UBATCH_SIZE`, `CACHE_TYPE_K`, `CACHE_TYPE_V`, + `SPEC_DRAFT_TYPE_K`, `SPEC_DRAFT_TYPE_V`, `EXTRA_ARGS`, and + `SPEC_EXTRA_ARGS` environment overrides for + controlled sweeps while defaulting to the promoted `n-max 4`, `n-min 0`, + `p-min 0.0`, `p-split 0.10`, q4 KV, `-t 16`, `-tb 32`, `-b 512`, and + `-ub 512` settings. + - `scripts/check-rocmfp4-quant-regression.sh` runs quant correctness plus + CPU quantizer, dequantizer, and vec-dot cycle ceilings for both ROCmFP4 + block layouts. It checks normal and imatrix quantization, and now also + protects CPU fallback dequant/vec-dot paths so decode-helper experiments + cannot pass while slowing non-GPU fallbacks. Recent pass after keeping the + `isfinite()` finite-block scoring promotion and adding the weighted/imatrix + finite scorer: dual-scale quant `3844.38`, FAST quant `3582.57`, dual + dequant `33.59`, FAST dequant `33.13`, dual vec-dot `29.96`, FAST vec-dot + `27.03`, dual imatrix `5587.43`, and FAST imatrix `4447.32` cycles / 32 + values. + - `scripts/check-rocmfp4-vulkan-runtime-regression.sh` measures focused + Vulkan `MUL_MAT` runtime for ROCmFP4 FAST and dual-scale layouts, catching + shader regressions that can be hidden by end-to-end decode noise. The + guard covers `n=1`, `n=2`, `n=4`, and `n=8` for both layouts. Recent + tightened serial pass measured FAST `55.82`, `71.59`, `105.10`, and + `163.41` us/run for `n=1/2/4/8`, and dual-scale `64.87`, `83.24`, + `118.56`, and `194.27` us/run for `n=1/2/4/8`. The previous serial pass + after adding the shared ROCmFP4 UE4M3 + scale LUT was FAST `53.66`, `71.67`, `105.14` us/run and dual-scale + `65.05`, `83.07`, `122.70` us/run for the same shapes. + - `scripts/check-rocmfp4-vulkan-cpy-regression.sh` measures Vulkan0 CPY for + `F32/F16/BF16 -> Q4_0_ROCMFP4`, + `F32/F16/BF16 -> Q4_0_ROCMFP4_FAST`, and the matching quant-to-F32 + dequant copy paths on the same large shape used by the ROCm CPY guard. + It is included in the all-regression harness to catch accidental Vulkan + copy fallback or shader-routing regressions. + - `scripts/check-rocmfp4-rocm-runtime-regression.sh` measures the same + focused `MUL_MAT` shapes on ROCm0. The guard covers `n=1`, `n=2`, `n=4`, + and `n=8` for both FAST and dual-scale ROCmFP4 so MTP-style multi-column + regressions are caught instead of only protecting single-token matvec. + Recent tightened serial pass after extending the RDNA3.5 two-warp route + through `n=2` measured FAST `51.66`, `66.56`, `101.27`, and `168.84` + us/run for `n=1/2/4/8`, and dual-scale `53.66`, `58.40`, `87.32`, and + `148.26` us/run for `n=1/2/4/8`. The `n=8` guard was added after a + rejected wide-rows candidate regressed dual-scale `n=8` to `1382.72` + us/run. + - `scripts/check-rocmfp4-rocm-cpy-regression.sh` measures ROCm0 CPY for + `F32/F16/BF16 -> Q4_0_ROCMFP4`, + `F32/F16/BF16 -> Q4_0_ROCMFP4_FAST`, and the matching quant-to-F32 + dequant copy paths on a large `8192x512x2` shape. Recent serial-gate pass + after direct FAST value scoring: F32/F16/BF16-to-dual `1111.73`, + `1008.69`, `1006.93` us/run, dual-to-F32 `182.25` us/run, + F32/F16/BF16-to-FAST `1047.21`, `950.93`, `951.00` us/run, and + FAST-to-F32 `170.36` us/run. + - `scripts/check-rocmfp4-all-regression.sh` runs the promoted-gain gate: + quant, Vulkan runtime, Vulkan CPY, ROCm runtime, ROCm FlashAttention, + ROCm CPY, and Qwen MTP guards serially, then checks ROCm KFD PIDs. Runtime + microbenchmarks should use this serial path rather than being run in + parallel with other GPU/UMA workloads. DeepSeek is not part of this + promoted-gain gate because no reproducible DeepSeek speedup has been + established; run the separate compatibility smoke only when explicitly + needed with `INCLUDE_DEEPSEEK_SMOKE=1`. The K/Q block-pair FlashAttention + serial pass held Qwen3.6 27B MTP at `33.2 tok/s` short and `27.7 tok/s` + sustained, with no KFD PIDs left running. The same pass measured ROCm + runtime `MUL_MAT` at FAST `54.16` / `66.13` / `101.73` / `178.36` us and + dual-scale `51.65` / `57.98` / `88.33` / `148.48` us for `n=1/2/4/8`, + and ROCm FlashAttention at `81.62` / `78.13` us for 64d dual-scale / FAST + and `228.58` / `199.32` us for Qwen-style 128d dual-scale / FAST. + - The MTP `accept()` path now skips re-copying `pending_h` when acceptance + lands on the final verify row already staged by `process()`. The focused + Qwen MTP guard held `33.8 tok/s` short and `27.9 tok/s` sustained after + this host-side cleanup. + - The MTP `process()` path now stores only the non-final verify rows needed + for partial-accept rollback, while copying the final target hidden row + directly to `pending_h` for the next MTP step. The focused Qwen MTP guard + held `33.9 tok/s` short and `27.9 tok/s` sustained, and the full serial + gate held `33.7 tok/s` short and `27.9 tok/s` sustained after this cleanup. + - The simple draft and MTP draft paths now return before + `llama_decode(ctx_dft, batch)` when no sequence is actively drafting. The + focused Qwen MTP guard held `33.8 tok/s` short and `28.0 tok/s` sustained, + and the full serial gate held `33.7 tok/s` short and `27.9 tok/s` + sustained after this idle/no-draft cleanup. + - The simple draft path now uses a reusable `uint8_t` drafting-state buffer + instead of allocating `std::vector` per draft call. The focused Qwen + MTP guard held `33.7 tok/s` short and `28.0 tok/s` sustained, and the full + serial gate held `33.8 tok/s` short and `27.9 tok/s` sustained. + - The MTP draft loop now skips its debug-candidate loop entirely unless + debug logging is enabled, hoists the debug verbosity state once per draft + call, and uses direct vector indexing in the hot per-token path. The + focused Qwen MTP guard held `33.8 tok/s` short and `27.9 tok/s` sustained + after this cleanup, so it is kept as a low-risk host-side simplification + rather than claimed as a decode-speed gain. + - The simple and MTP draft paths now use a shared direct one-sequence batch + append helper instead of `common_batch_add(..., { seq_id }, ...)` in the + hot speculative path. This avoids a temporary sequence-id container without + changing multi-sequence batch behavior elsewhere. The focused Qwen MTP + guard held `33.9 tok/s` short and `27.9 tok/s` sustained, and the full + serial gate held `33.8 tok/s` short and `27.9 tok/s` sustained. + - The MTP verify-row buffer now reserves for the configured draft depth and + only grows when needed instead of shrinking/resizing on every verification + pass. The focused Qwen MTP guard held `33.8 tok/s` short and `27.9 tok/s` + sustained, and the full serial gate held `33.9 tok/s` short and + `27.9 tok/s` sustained with no KFD PIDs left running. + - The MTP `process()` path now copies retained verification hidden rows from + the target embedding buffer in one contiguous `memcpy()` instead of one + `llama_get_embeddings_pre_norm_ith()` plus `memcpy()` per row. Rollback + behavior is unchanged. The focused Qwen3.6 27B MTP guard held + `33.7 tok/s` short and `27.9 tok/s` sustained, and the 35B A3B guard held + `104.1 tok/s` short and `89.2 tok/s` sustained. The same build later + passed the full serial all-regression gate with `INCLUDE_QWEN35_A3B_GUARD=1`, + including Qwen3.6 27B MTP at `33.8` / `27.9 tok/s` and Qwen3.6 35B A3B MTP + at `104.1` / `89.3 tok/s`. + - A matching contiguous-pointer cleanup inside the MTP `draft()` loop was + tested and rejected. It replaced per-row + `llama_get_embeddings_pre_norm_ith(ctx_dft, i_batch)` calls with one + `llama_get_embeddings_pre_norm(ctx_dft)` pointer per draft decode + iteration. The 27B MTP guard held at `33.8` / `27.9 tok/s`, but the 35B A3B + repeat measured `104.3` / `88.7` and `104.3` / `89.2 tok/s`, below the + promoted `89.3 tok/s` sustained band, so the code change was removed. + - A single-sequence MTP `draft()` fast path was tested and rejected on + 2026-05-25. It removed the active-sequence bookkeeping loop for the common + `n_seq == 1` case and passed the dense 27B guard at `33.7` / `28.0 tok/s`, + but the 35B A3B sustained guard collapsed to `25.7 tok/s` despite a + passing `103.1 tok/s` short check. Reverting that path restored the 35B A3B + guard to `104.3` / `90.3 tok/s`, so the shared multi-sequence draft loop + remains the promoted implementation. + - A dual-scale-only finite-pack CPU quantizer shortcut was tested and + rejected on 2026-05-24. It passed correctness, but even after isolating + the shared scale chooser it regressed the protected FAST quant path + (`q4_0_rocmfp4_fast` normal quant rose into the `3882`-`4022` cycles / 32 + values band versus the clean `3623` cycles / 32 values baseline). The + final packing loops therefore remain on the guarded nearest-code helper. +- The Strix ROCmFP4 build script now builds the quant regression test + binaries (`test-quantize-fns`, `test-quantize-perf`, and + `test-backend-ops`) alongside `llama-cli`, `llama-quantize`, and + `llama-bench`, so quant and runtime guards work from a clean build + directory. +- `test-quantize-perf --imatrix` benchmarks the ROCmFP4 quality/coherence + path through `ggml_quantize_chunk(..., imatrix)` with synthetic importance + weights. The quant regression guard includes this mode to reject imatrix-side + CPU changes that would not show up in the normal non-imatrix timing. + +High-upside Strix-specific work: +- The next significant speed step is likely a fused ROCmFP4 decode-and-dot + ROCm matvec path. The current ROCmFP4 wins come from compact blocks and + avoiding generic decode/copy overhead; a fused path would keep Codebook10 + values and UE4M3 scales in registers/shared memory through the dot product + instead of materializing wider intermediates. +- A ROCmFP4-aware long-context attention path is the second major target. The + long-context limit is mostly KV traffic, FlashAttention shape efficiency, and + memory bandwidth. The current dual-scale layout protects coherence, but it is + not itself a large speed lever unless the attention kernels consume it + directly and efficiently. +- MTP target/draft overlap is a possible scheduler project, not a flag. The + current common MTP flow is serial on one Strix Halo ROCm device. Real overlap + would need scheduler changes plus acceptance/correctness guards, and it + should only be promoted if it beats the sustained Qwen 262k guard. +- Coherence should stay protected by tensor-aware ROCmFP4 profiles: dual-scale + for sensitive tensors and FAST only where guarded model tests show no + sustained decode or quality regression. Pure speed-only profiles are not + promoted in this tree without the serial regression gate. + +Hardware note: +- This is a special AMD-targeted ggml/llama.cpp quantization and backend + path. It includes custom Vulkan and ROCm/HIP handling for the new GGUF + types, but it is not yet a native rocWMMA FP4 tensor-core implementation. + Current speed gains come from the compact block layout and backend decode + paths; deeper rocWMMA/cooperative-matrix work is future optimization work. +- NVIDIA CUDA is disabled in the Strix-FP4 build (`-DGGML_CUDA=OFF`). Some + upstream llama.cpp HIP backend sources still live under + `ggml/src/ggml-cuda` and are compiled by HIP for AMD, but the ROCmFP4-owned + helper code and user-facing build/run path are ROCm/HIP/Vulkan targeted. + This tree also accepts `GGML_HIP_ENABLE_UNIFIED_MEMORY=1` as the AMD-named + alias for the upstream unified-memory switch. +- The bundled rocWMMA 7.1.0 headers expose gfx12 WMMA paths for FP8/BF8 and + integer 8-bit inputs, but no native FP4 input type or FP4 WMMA/MFMA builtin + is visible locally. A true matrix-core ROCmFP4 path therefore needs a + measured unpack/convert strategy first, such as ROCmFP4 Codebook10 to int8 + WMMA or FP8 WMMA tiles, before claiming native FP4 tensor-core execution. +- rocWMMA FlashAttention is intentionally opt-in via + `GGML_HIP_ROCWMMA_FATTN=ON scripts/build-strix-rocmfp4-mtp.sh`. + It currently compiles with the local rocWMMA headers in + `/path/to/third_party/rocWMMA`, but the Strix Halo benchmark + regressed sustained Qwen MTP decode (`23.3 tok/s` vs the promoted + `26.2 tok/s` default HIP FlashAttention path), so the default build keeps it + disabled. A follow-up on the 35B A3B ROCmFP4 MTP guard measured only + `99.7` / `76.1 tok/s` versus the promoted `104.4` / `89.3` band, so this + remains true for the MoE-heavy profile as well. +- TurboQuant and TriAttention are not present as runtime flags in this isolated + tree. Integrating them would require a source-level merge into `strix-fp4`; + do not claim support or promote them unless they beat the serial ROCmFP4 + regression gate. diff --git a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.c b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.c new file mode 100644 index 000000000..bccefcec1 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.c @@ -0,0 +1,797 @@ +#define GGML_COMMON_DECL_C +#include "../src/ggml-common.h" + +#include "rocmfp4.h" + +#include +#include +#include + +// ggml-base is compiled architecture-neutral (no -mavx2), so SIMD for the hot +// CPU dot product is enabled per-function via a target attribute plus a runtime +// CPU check. This keeps the AVX2 path in one translation unit without exporting +// internals to ggml-cpu. Non-GNU/non-x86 builds fall back to the scalar loop. +#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) && !defined(ROCMFP4_NO_AVX2) +#include +#define ROCMFP4_X86_AVX2_DISPATCH 1 +#endif + +// ROCmFP4 stores a signed integer FP4-like codebook at half-scale. It is +// E2M1-derived, but the largest magnitude is retuned from 12 to 10 after +// sampling Qwen3 dense tensors; this reduces outlier pull without changing the +// packed 4-bit layout or integer dot-product path. +static const int8_t rocmfp4_codebook[16] = { + 0, 1, 2, 3, 4, 6, 8, 10, + 0, -1, -2, -3, -4, -6, -8,-10, +}; + +static inline int8_t rocmfp4_decode(uint8_t q) { + q &= 0x0f; + const int mag3 = q & 0x07; + const int mag = mag3 <= 4 ? mag3 : 2*mag3 - 4; + return (q & 0x08) ? -mag : mag; +} + +static inline int8_t rocmfp4_decode_table(uint8_t q) { + return rocmfp4_codebook[q & 0x0f]; +} + +// Finite unsigned E4M3 scale bytes decoded to the half-scale values used by +// ROCmFP4. Keeping this as a table avoids rebuilding identical FP32 values for +// every candidate during exhaustive scale search. +#define ROCMFP4_SCALE_SUB(M) ((M) * 0x1p-10f) +#define ROCMFP4_SCALE_E1(M) ((8 + (M)) * 0x1p-10f) +#define ROCMFP4_SCALE_E2(M) ((8 + (M)) * 0x1p-9f) +#define ROCMFP4_SCALE_E3(M) ((8 + (M)) * 0x1p-8f) +#define ROCMFP4_SCALE_E4(M) ((8 + (M)) * 0x1p-7f) +#define ROCMFP4_SCALE_E5(M) ((8 + (M)) * 0x1p-6f) +#define ROCMFP4_SCALE_E6(M) ((8 + (M)) * 0x1p-5f) +#define ROCMFP4_SCALE_E7(M) ((8 + (M)) * 0x1p-4f) +#define ROCMFP4_SCALE_E8(M) ((8 + (M)) * 0x1p-3f) +#define ROCMFP4_SCALE_E9(M) ((8 + (M)) * 0x1p-2f) +#define ROCMFP4_SCALE_E10(M) ((8 + (M)) * 0x1p-1f) +#define ROCMFP4_SCALE_E11(M) ((8 + (M)) * 0x1p0f) +#define ROCMFP4_SCALE_E12(M) ((8 + (M)) * 0x1p1f) +#define ROCMFP4_SCALE_E13(M) ((8 + (M)) * 0x1p2f) +#define ROCMFP4_SCALE_E14(M) ((8 + (M)) * 0x1p3f) +#define ROCMFP4_SCALE_E15(M) ((8 + (M)) * 0x1p4f) + +static const float rocmfp4_scale_ue4m3_half[127] = { + ROCMFP4_SCALE_SUB(0), ROCMFP4_SCALE_SUB(1), ROCMFP4_SCALE_SUB(2), ROCMFP4_SCALE_SUB(3), + ROCMFP4_SCALE_SUB(4), ROCMFP4_SCALE_SUB(5), ROCMFP4_SCALE_SUB(6), ROCMFP4_SCALE_SUB(7), + ROCMFP4_SCALE_E1(0), ROCMFP4_SCALE_E1(1), ROCMFP4_SCALE_E1(2), ROCMFP4_SCALE_E1(3), + ROCMFP4_SCALE_E1(4), ROCMFP4_SCALE_E1(5), ROCMFP4_SCALE_E1(6), ROCMFP4_SCALE_E1(7), + ROCMFP4_SCALE_E2(0), ROCMFP4_SCALE_E2(1), ROCMFP4_SCALE_E2(2), ROCMFP4_SCALE_E2(3), + ROCMFP4_SCALE_E2(4), ROCMFP4_SCALE_E2(5), ROCMFP4_SCALE_E2(6), ROCMFP4_SCALE_E2(7), + ROCMFP4_SCALE_E3(0), ROCMFP4_SCALE_E3(1), ROCMFP4_SCALE_E3(2), ROCMFP4_SCALE_E3(3), + ROCMFP4_SCALE_E3(4), ROCMFP4_SCALE_E3(5), ROCMFP4_SCALE_E3(6), ROCMFP4_SCALE_E3(7), + ROCMFP4_SCALE_E4(0), ROCMFP4_SCALE_E4(1), ROCMFP4_SCALE_E4(2), ROCMFP4_SCALE_E4(3), + ROCMFP4_SCALE_E4(4), ROCMFP4_SCALE_E4(5), ROCMFP4_SCALE_E4(6), ROCMFP4_SCALE_E4(7), + ROCMFP4_SCALE_E5(0), ROCMFP4_SCALE_E5(1), ROCMFP4_SCALE_E5(2), ROCMFP4_SCALE_E5(3), + ROCMFP4_SCALE_E5(4), ROCMFP4_SCALE_E5(5), ROCMFP4_SCALE_E5(6), ROCMFP4_SCALE_E5(7), + ROCMFP4_SCALE_E6(0), ROCMFP4_SCALE_E6(1), ROCMFP4_SCALE_E6(2), ROCMFP4_SCALE_E6(3), + ROCMFP4_SCALE_E6(4), ROCMFP4_SCALE_E6(5), ROCMFP4_SCALE_E6(6), ROCMFP4_SCALE_E6(7), + ROCMFP4_SCALE_E7(0), ROCMFP4_SCALE_E7(1), ROCMFP4_SCALE_E7(2), ROCMFP4_SCALE_E7(3), + ROCMFP4_SCALE_E7(4), ROCMFP4_SCALE_E7(5), ROCMFP4_SCALE_E7(6), ROCMFP4_SCALE_E7(7), + ROCMFP4_SCALE_E8(0), ROCMFP4_SCALE_E8(1), ROCMFP4_SCALE_E8(2), ROCMFP4_SCALE_E8(3), + ROCMFP4_SCALE_E8(4), ROCMFP4_SCALE_E8(5), ROCMFP4_SCALE_E8(6), ROCMFP4_SCALE_E8(7), + ROCMFP4_SCALE_E9(0), ROCMFP4_SCALE_E9(1), ROCMFP4_SCALE_E9(2), ROCMFP4_SCALE_E9(3), + ROCMFP4_SCALE_E9(4), ROCMFP4_SCALE_E9(5), ROCMFP4_SCALE_E9(6), ROCMFP4_SCALE_E9(7), + ROCMFP4_SCALE_E10(0), ROCMFP4_SCALE_E10(1), ROCMFP4_SCALE_E10(2), ROCMFP4_SCALE_E10(3), + ROCMFP4_SCALE_E10(4), ROCMFP4_SCALE_E10(5), ROCMFP4_SCALE_E10(6), ROCMFP4_SCALE_E10(7), + ROCMFP4_SCALE_E11(0), ROCMFP4_SCALE_E11(1), ROCMFP4_SCALE_E11(2), ROCMFP4_SCALE_E11(3), + ROCMFP4_SCALE_E11(4), ROCMFP4_SCALE_E11(5), ROCMFP4_SCALE_E11(6), ROCMFP4_SCALE_E11(7), + ROCMFP4_SCALE_E12(0), ROCMFP4_SCALE_E12(1), ROCMFP4_SCALE_E12(2), ROCMFP4_SCALE_E12(3), + ROCMFP4_SCALE_E12(4), ROCMFP4_SCALE_E12(5), ROCMFP4_SCALE_E12(6), ROCMFP4_SCALE_E12(7), + ROCMFP4_SCALE_E13(0), ROCMFP4_SCALE_E13(1), ROCMFP4_SCALE_E13(2), ROCMFP4_SCALE_E13(3), + ROCMFP4_SCALE_E13(4), ROCMFP4_SCALE_E13(5), ROCMFP4_SCALE_E13(6), ROCMFP4_SCALE_E13(7), + ROCMFP4_SCALE_E14(0), ROCMFP4_SCALE_E14(1), ROCMFP4_SCALE_E14(2), ROCMFP4_SCALE_E14(3), + ROCMFP4_SCALE_E14(4), ROCMFP4_SCALE_E14(5), ROCMFP4_SCALE_E14(6), ROCMFP4_SCALE_E14(7), + ROCMFP4_SCALE_E15(0), ROCMFP4_SCALE_E15(1), ROCMFP4_SCALE_E15(2), ROCMFP4_SCALE_E15(3), + ROCMFP4_SCALE_E15(4), ROCMFP4_SCALE_E15(5), ROCMFP4_SCALE_E15(6), +}; + +#undef ROCMFP4_SCALE_SUB +#undef ROCMFP4_SCALE_E1 +#undef ROCMFP4_SCALE_E2 +#undef ROCMFP4_SCALE_E3 +#undef ROCMFP4_SCALE_E4 +#undef ROCMFP4_SCALE_E5 +#undef ROCMFP4_SCALE_E6 +#undef ROCMFP4_SCALE_E7 +#undef ROCMFP4_SCALE_E8 +#undef ROCMFP4_SCALE_E9 +#undef ROCMFP4_SCALE_E10 +#undef ROCMFP4_SCALE_E11 +#undef ROCMFP4_SCALE_E12 +#undef ROCMFP4_SCALE_E13 +#undef ROCMFP4_SCALE_E14 +#undef ROCMFP4_SCALE_E15 + +static inline float rocmfp4_ue4m3_to_fp32_half(uint8_t e) { + return e <= 0x7e ? rocmfp4_scale_ue4m3_half[e] : 0.0f; +} + +static inline uint8_t rocmfp4_best_index_scaled_finite(float x, float inv_scale_half) { + // Exact nearest-neighbor thresholds for Codebook10: + // 0, +/-1, +/-2, +/-3, +/-4, +/-6, +/-8, +/-10 + // Ties intentionally choose the lower-magnitude code, matching the former + // linear scan because the positive codes and zero appear first. + const float a = fabsf(x * inv_scale_half); + if (a <= 0.5f) { + return 0; + } + + const bool neg = x < 0.0f; + if (a <= 1.5f) { + return neg ? 9 : 1; + } + if (a <= 2.5f) { + return neg ? 10 : 2; + } + if (a <= 3.5f) { + return neg ? 11 : 3; + } + if (a <= 5.0f) { + return neg ? 12 : 4; + } + if (a <= 7.0f) { + return neg ? 13 : 5; + } + if (a <= 9.0f) { + return neg ? 14 : 6; + } + + return neg ? 15 : 7; +} + +static inline uint8_t rocmfp4_best_index_scaled(float x, float inv_scale_half) { + if (!isfinite(x)) { + return 0; + } + + return rocmfp4_best_index_scaled_finite(x, inv_scale_half); +} + +// Fused best-index + decode used only inside the exhaustive scale search. The +// scale search re-scans every block element for every candidate scale byte, so +// avoiding the code -> decode round-trip on the hottest quantize path matters. +// Returns the same signed Codebook10 magnitude that +// rocmfp4_decode(rocmfp4_best_index_scaled_finite(x, inv_scale_half)) produces, +// so quantized output is bit-identical to the previous path. +static inline float rocmfp4_decoded_mag_scaled_finite(float x, float inv_scale_half) { + const float a = fabsf(x * inv_scale_half); + + float mag; + if (a <= 0.5f) { + mag = 0.0f; + } else if (a <= 1.5f) { + mag = 1.0f; + } else if (a <= 2.5f) { + mag = 2.0f; + } else if (a <= 3.5f) { + mag = 3.0f; + } else if (a <= 5.0f) { + mag = 4.0f; + } else if (a <= 7.0f) { + mag = 6.0f; + } else if (a <= 9.0f) { + mag = 8.0f; + } else { + mag = 10.0f; + } + + return x < 0.0f ? -mag : mag; +} + +static inline float rocmfp4_decoded_mag_scaled(float x, float inv_scale_half) { + if (!isfinite(x)) { + return 0.0f; + } + + return rocmfp4_decoded_mag_scaled_finite(x, inv_scale_half); +} + +static inline bool rocmfp4_scale_is_valid(uint8_t e) { + // ROCmFP4 scale bytes are unsigned finite E4M3 values. 0x7f is NaN in the + // unsigned encoding and values with the sign bit set are not valid scales. + return e <= 0x7e; +} + +static float rocmfp4_block_mse_for_scale_unweighted( + const float * x, int n, int e, float best_err) { + const float scale_half = rocmfp4_ue4m3_to_fp32_half((uint8_t) e); + const float inv_scale_half = 1.0f / scale_half; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + const float y = rocmfp4_decoded_mag_scaled(x[i], inv_scale_half) * scale_half; + const float d = x[i] - y; + + err += d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static float rocmfp4_block_mse_for_scale_unweighted_finite( + const float * x, int n, int e, float best_err) { + const float scale_half = rocmfp4_ue4m3_to_fp32_half((uint8_t) e); + const float inv_scale_half = 1.0f / scale_half; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + const float y = rocmfp4_decoded_mag_scaled_finite(x[i], inv_scale_half) * scale_half; + const float d = x[i] - y; + + err += d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static float rocmfp4_block_mse_for_scale_weighted( + const float * x, int n, const float * mse_weights, int e, float best_err) { + const float scale_half = rocmfp4_ue4m3_to_fp32_half((uint8_t) e); + const float inv_scale_half = 1.0f / scale_half; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + const float y = rocmfp4_decoded_mag_scaled(x[i], inv_scale_half) * scale_half; + const float d = x[i] - y; + + err += mse_weights[i]*d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static float rocmfp4_block_mse_for_scale_weighted_finite( + const float * x, int n, const float * mse_weights, int e, float best_err) { + const float scale_half = rocmfp4_ue4m3_to_fp32_half((uint8_t) e); + const float inv_scale_half = 1.0f / scale_half; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + const float y = rocmfp4_decoded_mag_scaled_finite(x[i], inv_scale_half) * scale_half; + const float d = x[i] - y; + + err += mse_weights[i]*d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static void rocmfp4_prepare_mse_weights( + float * dst, const float * x, int n, const float * quant_weights, float sigma2, + float * max_abs, float * max_abs_weight, bool * all_finite) { + *max_abs = 0.0f; + *max_abs_weight = 0.0f; + *all_finite = true; + + for (int i = 0; i < n; ++i) { + const float qw = quant_weights[i]; + const float ax = fabsf(x[i]); + const float weight = isfinite(qw) && qw > 0.0f ? qw * sqrtf(sigma2 + x[i]*x[i]) : 0.0f; + *all_finite = *all_finite && isfinite(x[i]); + + if (ax > *max_abs) { + *max_abs = ax; + *max_abs_weight = weight; + } else if (ax == *max_abs && weight > *max_abs_weight) { + *max_abs_weight = weight; + } + + // Match llama.cpp's imatrix weighting style for Q4_0: calibration + // importance is scaled by row energy so large activations remain protected. + dst[i] = weight; + } +} + +static int rocmfp4_nearest_scale_ue4m3(float target_scale_half) { + if (!(target_scale_half > 0.0f) || !isfinite(target_scale_half)) { + return 1; + } + + int lo = 1; + int hi = 126; + while (lo < hi) { + const int mid = lo + (hi - lo) / 2; + if (rocmfp4_ue4m3_to_fp32_half((uint8_t) mid) < target_scale_half) { + lo = mid + 1; + } else { + hi = mid; + } + } + + if (lo == 1) { + return 1; + } + + const float hi_scale = rocmfp4_ue4m3_to_fp32_half((uint8_t) lo); + const float lo_scale = rocmfp4_ue4m3_to_fp32_half((uint8_t) (lo - 1)); + + // Match the former ascending nearest scan: exact midpoint ties keep the + // lower scale byte. + return (target_scale_half - lo_scale <= hi_scale - target_scale_half) ? lo - 1 : lo; +} + +static uint8_t rocmfp4_choose_scale_ue4m3_exhaustive_unweighted( + const float * x, int n, float max_abs, bool all_finite) { + const int start_e = rocmfp4_nearest_scale_ue4m3(max_abs / 10.0f); + + int best_e = 0; + float best_err = FLT_MAX; + bool lower_done = false; + + for (int delta = 0; delta <= 125; ++delta) { + const int e0 = start_e - delta; + if (!lower_done && e0 >= 1 && e0 <= 126) { + const float scale_half = rocmfp4_ue4m3_to_fp32_half((uint8_t) e0); + const float clip_delta = max_abs - 10.0f*scale_half; + if (clip_delta > 0.0f && clip_delta*clip_delta > best_err) { + lower_done = true; + } else { + const float err = all_finite ? + rocmfp4_block_mse_for_scale_unweighted_finite(x, n, e0, best_err) : + rocmfp4_block_mse_for_scale_unweighted(x, n, e0, best_err); + if (err < best_err || (err == best_err && e0 < best_e)) { + best_err = err; + best_e = e0; + } + } + } + + const int e1 = start_e + delta; + if (delta != 0 && e1 >= 1 && e1 <= 126) { + const float err = all_finite ? + rocmfp4_block_mse_for_scale_unweighted_finite(x, n, e1, best_err) : + rocmfp4_block_mse_for_scale_unweighted(x, n, e1, best_err); + if (err < best_err || (err == best_err && e1 < best_e)) { + best_err = err; + best_e = e1; + } + } + + if ((lower_done || e0 <= 1) && e1 >= 126) { + break; + } + } + + return (uint8_t) best_e; +} + +static uint8_t rocmfp4_choose_scale_ue4m3_exhaustive_weighted( + const float * x, int n, const float * mse_weights, float max_abs, float max_abs_weight, bool all_finite) { + const int start_e = rocmfp4_nearest_scale_ue4m3(max_abs / 10.0f); + + int best_e = 0; + float best_err = FLT_MAX; + bool lower_done = false; + + for (int delta = 0; delta <= 125; ++delta) { + const int e0 = start_e - delta; + if (!lower_done && e0 >= 1 && e0 <= 126) { + const float scale_half = rocmfp4_ue4m3_to_fp32_half((uint8_t) e0); + const float clip_delta = max_abs - 10.0f*scale_half; + if (max_abs_weight > 0.0f && clip_delta > 0.0f && max_abs_weight*clip_delta*clip_delta > best_err) { + lower_done = true; + } else { + const float err = all_finite ? + rocmfp4_block_mse_for_scale_weighted_finite(x, n, mse_weights, e0, best_err) : + rocmfp4_block_mse_for_scale_weighted(x, n, mse_weights, e0, best_err); + if (err < best_err || (err == best_err && e0 < best_e)) { + best_err = err; + best_e = e0; + } + } + } + + const int e1 = start_e + delta; + if (delta != 0 && e1 >= 1 && e1 <= 126) { + const float err = all_finite ? + rocmfp4_block_mse_for_scale_weighted_finite(x, n, mse_weights, e1, best_err) : + rocmfp4_block_mse_for_scale_weighted(x, n, mse_weights, e1, best_err); + if (err < best_err || (err == best_err && e1 < best_e)) { + best_err = err; + best_e = e1; + } + } + + if ((lower_done || e0 <= 1) && e1 >= 126) { + break; + } + } + + return (uint8_t) best_e; +} + +static uint8_t rocmfp4_choose_scale_ue4m3(const float * x, int n, const float * quant_weights, float sigma2) { + if (quant_weights) { + assert(n <= QK_ROCMFP4); + float mse_weights_buf[QK_ROCMFP4]; + float weighted_max_abs; + float max_abs_weight; + bool all_finite; + rocmfp4_prepare_mse_weights(mse_weights_buf, x, n, quant_weights, sigma2, &weighted_max_abs, &max_abs_weight, &all_finite); + if (!(weighted_max_abs > 0.0f) || !isfinite(weighted_max_abs)) { + return 0; + } + return rocmfp4_choose_scale_ue4m3_exhaustive_weighted(x, n, mse_weights_buf, weighted_max_abs, max_abs_weight, all_finite); + } + + float max_abs = 0.0f; + bool all_finite = true; + for (int i = 0; i < n; ++i) { + all_finite = all_finite && isfinite(x[i]); + const float ax = fabsf(x[i]); + if (ax > max_abs) { + max_abs = ax; + } + } + + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfp4_choose_scale_ue4m3_exhaustive_unweighted(x, n, max_abs, all_finite); +} + +static void rocmfp4_quantize_row_q4_0_weighted( + const float * GGML_RESTRICT x, block_rocmfp4 * GGML_RESTRICT y, int64_t k, const float * GGML_RESTRICT quant_weights) { + assert(k % QK_ROCMFP4 == 0); + + float sum_x2 = 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum_x2 += x[i]*x[i]; + } + const float sigma2 = sum_x2 / (float) k; + + const int64_t nb = k / QK_ROCMFP4; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP4; + const float * qw = quant_weights ? quant_weights + ib*QK_ROCMFP4 : NULL; + const uint8_t e0 = rocmfp4_choose_scale_ue4m3(xb, QK_ROCMFP4/2, qw, sigma2); + const uint8_t e1 = rocmfp4_choose_scale_ue4m3(xb + QK_ROCMFP4/2, QK_ROCMFP4/2, qw ? qw + QK_ROCMFP4/2 : NULL, sigma2); + const float scale_half0 = rocmfp4_ue4m3_to_fp32_half(e0); + const float scale_half1 = rocmfp4_ue4m3_to_fp32_half(e1); + const float inv_scale_half0 = scale_half0 > 0.0f ? 1.0f / scale_half0 : 0.0f; + const float inv_scale_half1 = scale_half1 > 0.0f ? 1.0f / scale_half1 : 0.0f; + + y[ib].e[0] = e0; + y[ib].e[1] = e1; + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q0 = rocmfp4_best_index_scaled(xb[j], inv_scale_half0); + const uint8_t q1 = rocmfp4_best_index_scaled(xb[j + QK_ROCMFP4/2], inv_scale_half1); + y[ib].qs[j] = q0 | (q1 << 4); + } + } +} + +static void rocmfp4_quantize_row_q4_0_fast_weighted( + const float * GGML_RESTRICT x, block_rocmfp4_fast * GGML_RESTRICT y, int64_t k, const float * GGML_RESTRICT quant_weights) { + assert(k % QK_ROCMFP4 == 0); + + float sum_x2 = 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum_x2 += x[i]*x[i]; + } + const float sigma2 = sum_x2 / (float) k; + + const int64_t nb = k / QK_ROCMFP4; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP4; + const float * qw = quant_weights ? quant_weights + ib*QK_ROCMFP4 : NULL; + const uint8_t e = rocmfp4_choose_scale_ue4m3(xb, QK_ROCMFP4, qw, sigma2); + const float scale_half = rocmfp4_ue4m3_to_fp32_half(e); + const float inv_scale_half = scale_half > 0.0f ? 1.0f / scale_half : 0.0f; + + y[ib].e = e; + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q0 = rocmfp4_best_index_scaled(xb[j], inv_scale_half); + const uint8_t q1 = rocmfp4_best_index_scaled(xb[j + QK_ROCMFP4/2], inv_scale_half); + y[ib].qs[j] = q0 | (q1 << 4); + } + } +} + +void rocmfp4_quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_rocmfp4 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP4 == 0); + + const int64_t nb = k / QK_ROCMFP4; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP4; + const uint8_t e0 = rocmfp4_choose_scale_ue4m3(xb, QK_ROCMFP4/2, NULL, 0.0f); + const uint8_t e1 = rocmfp4_choose_scale_ue4m3(xb + QK_ROCMFP4/2, QK_ROCMFP4/2, NULL, 0.0f); + const float scale_half0 = rocmfp4_ue4m3_to_fp32_half(e0); + const float scale_half1 = rocmfp4_ue4m3_to_fp32_half(e1); + const float inv_scale_half0 = scale_half0 > 0.0f ? 1.0f / scale_half0 : 0.0f; + const float inv_scale_half1 = scale_half1 > 0.0f ? 1.0f / scale_half1 : 0.0f; + + y[ib].e[0] = e0; + y[ib].e[1] = e1; + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q0 = rocmfp4_best_index_scaled(xb[j], inv_scale_half0); + const uint8_t q1 = rocmfp4_best_index_scaled(xb[j + QK_ROCMFP4/2], inv_scale_half1); + y[ib].qs[j] = q0 | (q1 << 4); + } + } +} + +void rocmfp4_quantize_row_q4_0_fast_ref(const float * GGML_RESTRICT x, block_rocmfp4_fast * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP4 == 0); + + const int64_t nb = k / QK_ROCMFP4; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP4; + const uint8_t e = rocmfp4_choose_scale_ue4m3(xb, QK_ROCMFP4, NULL, 0.0f); + const float scale_half = rocmfp4_ue4m3_to_fp32_half(e); + const float inv_scale_half = scale_half > 0.0f ? 1.0f / scale_half : 0.0f; + + y[ib].e = e; + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q0 = rocmfp4_best_index_scaled(xb[j], inv_scale_half); + const uint8_t q1 = rocmfp4_best_index_scaled(xb[j + QK_ROCMFP4/2], inv_scale_half); + y[ib].qs[j] = q0 | (q1 << 4); + } + } +} + +void rocmfp4_dequantize_row_q4_0(const block_rocmfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP4 == 0); + + const int64_t nb = k / QK_ROCMFP4; + for (int64_t ib = 0; ib < nb; ++ib) { + const float d0 = rocmfp4_ue4m3_to_fp32_half(x[ib].e[0]); + const float d1 = rocmfp4_ue4m3_to_fp32_half(x[ib].e[1]); + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + y[ib*QK_ROCMFP4 + j] = (float) rocmfp4_decode(x[ib].qs[j] & 0x0f) * d0; + y[ib*QK_ROCMFP4 + j + QK_ROCMFP4/2] = (float) rocmfp4_decode(x[ib].qs[j] >> 4) * d1; + } + } +} + +void rocmfp4_dequantize_row_q4_0_fast(const block_rocmfp4_fast * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP4 == 0); + + const int64_t nb = k / QK_ROCMFP4; + for (int64_t ib = 0; ib < nb; ++ib) { + const float d = rocmfp4_ue4m3_to_fp32_half(x[ib].e); + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + y[ib*QK_ROCMFP4 + j] = (float) rocmfp4_decode(x[ib].qs[j] & 0x0f) * d; + y[ib*QK_ROCMFP4 + j + QK_ROCMFP4/2] = (float) rocmfp4_decode(x[ib].qs[j] >> 4) * d; + } + } +} + +void rocmfp4_quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + rocmfp4_quantize_row_q4_0_ref(x, (block_rocmfp4 *) y, k); +} + +void rocmfp4_quantize_row_q4_0_fast(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + rocmfp4_quantize_row_q4_0_fast_ref(x, (block_rocmfp4_fast *) y, k); +} + +size_t rocmfp4_quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { + const size_t row_size = ggml_row_size(GGML_TYPE_Q4_0_ROCMFP4, n_per_row); + + if (!imatrix) { + rocmfp4_quantize_row_q4_0_ref(src, (block_rocmfp4 *) dst, nrows*n_per_row); + return nrows * row_size; + } + + char * qrow = (char *) dst; + for (int64_t row = 0; row < nrows; ++row) { + rocmfp4_quantize_row_q4_0_weighted(src, (block_rocmfp4 *) qrow, n_per_row, imatrix); + src += n_per_row; + qrow += row_size; + } + + return nrows * row_size; +} + +size_t rocmfp4_quantize_q4_0_fast(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { + const size_t row_size = ggml_row_size(GGML_TYPE_Q4_0_ROCMFP4_FAST, n_per_row); + + if (!imatrix) { + rocmfp4_quantize_row_q4_0_fast_ref(src, (block_rocmfp4_fast *) dst, nrows*n_per_row); + return nrows * row_size; + } + + char * qrow = (char *) dst; + for (int64_t row = 0; row < nrows; ++row) { + rocmfp4_quantize_row_q4_0_fast_weighted(src, (block_rocmfp4_fast *) qrow, n_per_row, imatrix); + src += n_per_row; + qrow += row_size; + } + + return nrows * row_size; +} + +bool rocmfp4_validate_row_data(const void * data, size_t nbytes) { + if (nbytes % sizeof(block_rocmfp4) != 0) { + return false; + } + + const block_rocmfp4 * blocks = (const block_rocmfp4 *) data; + const size_t nblocks = nbytes / sizeof(block_rocmfp4); + for (size_t i = 0; i < nblocks; ++i) { + if (!rocmfp4_scale_is_valid(blocks[i].e[0]) || !rocmfp4_scale_is_valid(blocks[i].e[1])) { + return false; + } + } + + return true; +} + +bool rocmfp4_validate_row_data_fast(const void * data, size_t nbytes) { + if (nbytes % sizeof(block_rocmfp4_fast) != 0) { + return false; + } + + const block_rocmfp4_fast * blocks = (const block_rocmfp4_fast *) data; + const size_t nblocks = nbytes / sizeof(block_rocmfp4_fast); + for (size_t i = 0; i < nblocks; ++i) { + if (!rocmfp4_scale_is_valid(blocks[i].e)) { + return false; + } + } + + return true; +} + +#ifdef ROCMFP4_X86_AVX2_DISPATCH +__attribute__((target("avx2"))) +static inline int rocmfp4_hsum_i32_8_avx2(__m256i v) { + __m128i s = _mm_add_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256(v, 1)); + s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(1, 0, 3, 2))); + s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(2, 3, 0, 1))); + return _mm_cvtsi128_si32(s); +} + +// Decode one 32-weight block's low and high nibble streams through the +// Codebook10 table with a single PSHUFB, then integer-dot each against its half +// of the q8_0 block. Integer sums are order-independent, so sumi0/sumi1 match +// the scalar reference exactly and the float result is bit-identical. +__attribute__((target("avx2"))) +static inline void rocmfp4_block_isums_avx2( + const uint8_t * qs, const int8_t * q8, int * sumi0, int * sumi1) { + const __m128i tbl = _mm_loadu_si128((const __m128i *) rocmfp4_codebook); + const __m128i q = _mm_loadu_si128((const __m128i *) qs); + const __m128i lo = _mm_and_si128(q, _mm_set1_epi8(0x0F)); + const __m128i hi = _mm_and_si128(_mm_srli_epi16(q, 4), _mm_set1_epi8(0x0F)); + const __m128i dlo = _mm_shuffle_epi8(tbl, lo); + const __m128i dhi = _mm_shuffle_epi8(tbl, hi); + const __m128i ylo = _mm_loadu_si128((const __m128i *) q8); + const __m128i yhi = _mm_loadu_si128((const __m128i *) (q8 + QK_ROCMFP4/2)); + const __m256i pl = _mm256_madd_epi16(_mm256_cvtepi8_epi16(dlo), _mm256_cvtepi8_epi16(ylo)); + const __m256i ph = _mm256_madd_epi16(_mm256_cvtepi8_epi16(dhi), _mm256_cvtepi8_epi16(yhi)); + *sumi0 = rocmfp4_hsum_i32_8_avx2(pl); + *sumi1 = rocmfp4_hsum_i32_8_avx2(ph); +} + +__attribute__((target("avx2"))) +static void rocmfp4_vec_dot_q4_0_q8_0_avx2( + int nb, float * GGML_RESTRICT s, const block_rocmfp4 * GGML_RESTRICT x, const block_q8_0 * GGML_RESTRICT y) { + float sumf = 0.0f; + for (int ib = 0; ib < nb; ++ib) { + const float d0 = rocmfp4_ue4m3_to_fp32_half(x[ib].e[0]) * ggml_fp16_to_fp32(y[ib].d); + const float d1 = rocmfp4_ue4m3_to_fp32_half(x[ib].e[1]) * ggml_fp16_to_fp32(y[ib].d); + int sumi0, sumi1; + rocmfp4_block_isums_avx2(x[ib].qs, y[ib].qs, &sumi0, &sumi1); + sumf += d0 * (float) sumi0 + d1 * (float) sumi1; + } + *s = sumf; +} + +__attribute__((target("avx2"))) +static void rocmfp4_vec_dot_q4_0_fast_q8_0_avx2( + int nb, float * GGML_RESTRICT s, const block_rocmfp4_fast * GGML_RESTRICT x, const block_q8_0 * GGML_RESTRICT y) { + float sumf = 0.0f; + for (int ib = 0; ib < nb; ++ib) { + const float d = rocmfp4_ue4m3_to_fp32_half(x[ib].e) * ggml_fp16_to_fp32(y[ib].d); + int sumi0, sumi1; + rocmfp4_block_isums_avx2(x[ib].qs, y[ib].qs, &sumi0, &sumi1); + sumf += d * (float) (sumi0 + sumi1); + } + *s = sumf; +} +#endif // ROCMFP4_X86_AVX2_DISPATCH + +void rocmfp4_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + GGML_UNUSED(bs); + GGML_UNUSED(bx); + GGML_UNUSED(by); + assert(nrc == 1); + GGML_UNUSED(nrc); + assert(n % QK_ROCMFP4 == 0); + assert(QK_ROCMFP4 == QK8_0); + + const block_rocmfp4 * GGML_RESTRICT x = (const block_rocmfp4 *) vx; + const block_q8_0 * GGML_RESTRICT y = (const block_q8_0 *) vy; + + const int nb = n / QK_ROCMFP4; + +#ifdef ROCMFP4_X86_AVX2_DISPATCH + if (__builtin_cpu_supports("avx2")) { + rocmfp4_vec_dot_q4_0_q8_0_avx2(nb, s, x, y); + return; + } +#endif + + float sumf = 0.0f; + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = rocmfp4_ue4m3_to_fp32_half(x[ib].e[0]) * ggml_fp16_to_fp32(y[ib].d); + const float d1 = rocmfp4_ue4m3_to_fp32_half(x[ib].e[1]) * ggml_fp16_to_fp32(y[ib].d); + int sumi0 = 0; + int sumi1 = 0; + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q = x[ib].qs[j]; + sumi0 += rocmfp4_decode_table(q) * y[ib].qs[j]; + sumi1 += rocmfp4_decode_table(q >> 4) * y[ib].qs[j + QK_ROCMFP4/2]; + } + + sumf += d0 * (float) sumi0 + d1 * (float) sumi1; + } + + *s = sumf; +} + +void rocmfp4_vec_dot_q4_0_fast_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + GGML_UNUSED(bs); + GGML_UNUSED(bx); + GGML_UNUSED(by); + assert(nrc == 1); + GGML_UNUSED(nrc); + assert(n % QK_ROCMFP4 == 0); + assert(QK_ROCMFP4 == QK8_0); + + const block_rocmfp4_fast * GGML_RESTRICT x = (const block_rocmfp4_fast *) vx; + const block_q8_0 * GGML_RESTRICT y = (const block_q8_0 *) vy; + + const int nb = n / QK_ROCMFP4; + +#ifdef ROCMFP4_X86_AVX2_DISPATCH + if (__builtin_cpu_supports("avx2")) { + rocmfp4_vec_dot_q4_0_fast_q8_0_avx2(nb, s, x, y); + return; + } +#endif + + float sumf = 0.0f; + + for (int ib = 0; ib < nb; ++ib) { + const float d = rocmfp4_ue4m3_to_fp32_half(x[ib].e) * ggml_fp16_to_fp32(y[ib].d); + int sumi = 0; + + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q = x[ib].qs[j]; + sumi += rocmfp4_decode_table(q) * y[ib].qs[j]; + sumi += rocmfp4_decode_table(q >> 4) * y[ib].qs[j + QK_ROCMFP4/2]; + } + + sumf += d * (float) sumi; + } + + *s = sumf; +} diff --git a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.h b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.h new file mode 100644 index 000000000..9756f6ad4 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define QK_ROCMFP4 32 +#define QR_ROCMFP4 2 +#define QI_ROCMFP4 (QK_ROCMFP4 / (4 * QR_ROCMFP4)) +#define QS_ROCMFP4 32 + +// AMD-tuned compact layout: 16 bytes of packed E2M1-derived 4-bit codes, then +// one unsigned E4M3 scale byte per 16-weight half block. +typedef struct { + uint8_t qs[QK_ROCMFP4/2]; + uint8_t e[2]; +} block_rocmfp4; + +// Speed-focused layout: same 32 packed ROCmFP4 nibbles, but one UE4M3 scale +// for the whole block. This is a separate GGUF type so fast 4.25 BPW artifacts +// never alias the safer dual-scale format above. +typedef struct { + uint8_t qs[QK_ROCMFP4/2]; + uint8_t e; +} block_rocmfp4_fast; + +#if defined(__cplusplus) +static_assert(sizeof(block_rocmfp4) == QK_ROCMFP4/2 + 2*sizeof(uint8_t), "wrong rocmfp4 block size/padding"); +static_assert(sizeof(block_rocmfp4_fast) == QK_ROCMFP4/2 + sizeof(uint8_t), "wrong rocmfp4 fast block size/padding"); +#else +_Static_assert(sizeof(block_rocmfp4) == QK_ROCMFP4/2 + 2*sizeof(uint8_t), "wrong rocmfp4 block size/padding"); +_Static_assert(sizeof(block_rocmfp4_fast) == QK_ROCMFP4/2 + sizeof(uint8_t), "wrong rocmfp4 fast block size/padding"); +#endif + +GGML_API void rocmfp4_quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_rocmfp4 * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfp4_dequantize_row_q4_0(const block_rocmfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfp4_quantize_row_q4_0_fast_ref(const float * GGML_RESTRICT x, block_rocmfp4_fast * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfp4_dequantize_row_q4_0_fast(const block_rocmfp4_fast * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + +GGML_API void rocmfp4_quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +GGML_API size_t rocmfp4_quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void rocmfp4_quantize_row_q4_0_fast(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +GGML_API size_t rocmfp4_quantize_q4_0_fast(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API bool rocmfp4_validate_row_data(const void * data, size_t nbytes); +GGML_API bool rocmfp4_validate_row_data_fast(const void * data, size_t nbytes); + +GGML_API void rocmfp4_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +GGML_API void rocmfp4_vec_dot_q4_0_fast_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); + +#ifdef __cplusplus +} +#endif diff --git a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip.cu b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip.cu new file mode 100644 index 000000000..d2c9048c1 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip.cu @@ -0,0 +1,85 @@ +#include "rocmfp4.h" + +#include + +#include "rocmfp4_hip_scale.cuh" + +// Standalone ROCm/HIP dequant kernel for integration tests and future fused +// paths. One lane owns one packed byte and writes the matching low/high +// half-block values, so each byte is read once. +extern "C" __global__ void rocmfp4_dequantize_q4_0_f32_kernel( + const block_rocmfp4 * __restrict__ x, + float * __restrict__ y, + int64_t k) { + const int64_t packed_idx = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; + const int64_t nblocks = (k + QK_ROCMFP4 - 1) / QK_ROCMFP4; + const int64_t packed_count = nblocks * (QK_ROCMFP4/2); + + if (packed_idx >= packed_count) { + return; + } + + const int64_t ib = packed_idx / (QK_ROCMFP4/2); + const int tid = packed_idx - ib*(QK_ROCMFP4/2); + const int64_t base = ib*QK_ROCMFP4; + const uint8_t packed = x[ib].qs[tid]; + const float d0 = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e[0]); + const float d1 = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e[1]); + + if (base + tid < k) { + y[base + tid] = (float) rocmfp4_decode_i8(packed & 0x0f) * d0; + } + if (base + tid + QK_ROCMFP4/2 < k) { + y[base + tid + QK_ROCMFP4/2] = (float) rocmfp4_decode_i8(packed >> 4) * d1; + } +} + +extern "C" __global__ void rocmfp4_dequantize_q4_0_fast_f32_kernel( + const block_rocmfp4_fast * __restrict__ x, + float * __restrict__ y, + int64_t k) { + const int64_t packed_idx = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; + const int64_t nblocks = (k + QK_ROCMFP4 - 1) / QK_ROCMFP4; + const int64_t packed_count = nblocks * (QK_ROCMFP4/2); + + if (packed_idx >= packed_count) { + return; + } + + const int64_t ib = packed_idx / (QK_ROCMFP4/2); + const int tid = packed_idx - ib*(QK_ROCMFP4/2); + const int64_t base = ib*QK_ROCMFP4; + const uint8_t packed = x[ib].qs[tid]; + const float d = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e); + + if (base + tid < k) { + y[base + tid] = (float) rocmfp4_decode_i8(packed & 0x0f) * d; + } + if (base + tid + QK_ROCMFP4/2 < k) { + y[base + tid + QK_ROCMFP4/2] = (float) rocmfp4_decode_i8(packed >> 4) * d; + } +} + +extern "C" void rocmfp4_hip_dequantize_q4_0_to_f32( + const void * src, + float * dst, + int64_t k, + hipStream_t stream) { + const int64_t nblocks = (k + QK_ROCMFP4 - 1) / QK_ROCMFP4; + const int64_t packed_count = nblocks * (QK_ROCMFP4/2); + const dim3 block(256); + const dim3 grid((unsigned int) ((packed_count + block.x - 1) / block.x)); + rocmfp4_dequantize_q4_0_f32_kernel<<>>((const block_rocmfp4 *) src, dst, k); +} + +extern "C" void rocmfp4_hip_dequantize_q4_0_fast_to_f32( + const void * src, + float * dst, + int64_t k, + hipStream_t stream) { + const int64_t nblocks = (k + QK_ROCMFP4 - 1) / QK_ROCMFP4; + const int64_t packed_count = nblocks * (QK_ROCMFP4/2); + const dim3 block(256); + const dim3 grid((unsigned int) ((packed_count + block.x - 1) / block.x)); + rocmfp4_dequantize_q4_0_fast_f32_kernel<<>>((const block_rocmfp4_fast *) src, dst, k); +} diff --git a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh new file mode 100644 index 000000000..1d9106112 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh @@ -0,0 +1,79 @@ +#pragma once + +#include "rocmfp4_hip_scale.cuh" + +#include +#include + +#ifndef GGML_ROCMFP4_UNALIGNED_QS_DWORD_LOAD +#define GGML_ROCMFP4_UNALIGNED_QS_DWORD_LOAD 1 +#endif + +static __device__ __forceinline__ int rocmfp4_get_qs_i32(const void * x, const int & i32) { +#if defined(GGML_USE_HIP) && GGML_ROCMFP4_UNALIGNED_QS_DWORD_LOAD + return *((const int *) ((const uint8_t *) x + 4*i32)); +#else + const uint8_t * x8 = (const uint8_t *) x; + + int x32 = x8[4*i32 + 0] << 0; + x32 |= x8[4*i32 + 1] << 8; + x32 |= x8[4*i32 + 2] << 16; + x32 |= x8[4*i32 + 3] << 24; + + return x32; +#endif +} + +// AMD-specific fast path for expanding eight packed ROCmFP4 nibbles into two +// int32 DP4A operands. This encodes the Codebook10 table directly as four +// 32-bit constants: +// [0, 1, 2, 3], [4, 6, 8, 10], [0, -1, -2, -3], [-4, -6, -8, -10] +// Avoiding the table pointer keeps the ROCm/HIP MMVQ/MMQ hot path fully local +// to this format. Non-HIP builds still use llama.cpp's generic table expander. +static __device__ __forceinline__ int2 rocmfp4_get_int_from_codebook_16(const int & q4, const int8_t * fallback_table) { +#if defined(GGML_USE_HIP) + constexpr uint32_t values0 = 0x03020100u; + constexpr uint32_t values1 = 0x0a080604u; + constexpr uint32_t values2 = 0xfdfeff00u; + constexpr uint32_t values3 = 0xf6f8fafcu; + + const uint32_t q_even = q4; + const uint32_t q_odd = q4 >> 4; + + const uint32_t v_even_low = __builtin_amdgcn_perm(values1, values0, q_even & 0x07070707u); + const uint32_t v_odd_low = __builtin_amdgcn_perm(values1, values0, q_odd & 0x07070707u); + const uint32_t v_even_high = __builtin_amdgcn_perm(values3, values2, q_even & 0x07070707u); + const uint32_t v_odd_high = __builtin_amdgcn_perm(values3, values2, q_odd & 0x07070707u); + + const uint32_t mask_even = 0x03020100u | ((q_even & 0x08080808u) >> 1); + const uint32_t mask_odd = 0x03020100u | ((q_odd & 0x08080808u) >> 1); + + return make_int2( + __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even), + __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd)); +#else + return get_int_from_table_16(q4, fallback_table); +#endif +} + +// Variant for call sites that already selected either the low or high nibble +// stream and only need one DP4A operand. This avoids the extra odd/even table +// expansion work in ROCmFP4 FlashAttention K/V decode. +static __device__ __forceinline__ int rocmfp4_get_low_int_from_codebook_16(const int & q4, const int8_t * fallback_table) { +#if defined(GGML_USE_HIP) + constexpr uint32_t values0 = 0x03020100u; + constexpr uint32_t values1 = 0x0a080604u; + constexpr uint32_t values2 = 0xfdfeff00u; + constexpr uint32_t values3 = 0xf6f8fafcu; + + const uint32_t q = q4; + + const uint32_t v_low = __builtin_amdgcn_perm(values1, values0, q & 0x07070707u); + const uint32_t v_high = __builtin_amdgcn_perm(values3, values2, q & 0x07070707u); + const uint32_t mask = 0x03020100u | ((q & 0x08080808u) >> 1); + + return __builtin_amdgcn_perm(v_high, v_low, mask); +#else + return get_int_from_table_16(q4, fallback_table).x; +#endif +} diff --git a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_scale.cuh b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_scale.cuh new file mode 100644 index 000000000..19525a4fb --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_scale.cuh @@ -0,0 +1,157 @@ +#pragma once + +#include +#include +#include + +#ifndef GGML_ROCMFP4_USE_SCALE_LUT +#define GGML_ROCMFP4_USE_SCALE_LUT 0 +#endif + +#if defined(GGML_USE_HIP) && GGML_ROCMFP4_USE_SCALE_LUT +#define ROCMFP4_SCALE_SUB(M) ((M) * 0x1p-10f) +#define ROCMFP4_SCALE_E1(M) ((8 + (M)) * 0x1p-10f) +#define ROCMFP4_SCALE_E2(M) ((8 + (M)) * 0x1p-9f) +#define ROCMFP4_SCALE_E3(M) ((8 + (M)) * 0x1p-8f) +#define ROCMFP4_SCALE_E4(M) ((8 + (M)) * 0x1p-7f) +#define ROCMFP4_SCALE_E5(M) ((8 + (M)) * 0x1p-6f) +#define ROCMFP4_SCALE_E6(M) ((8 + (M)) * 0x1p-5f) +#define ROCMFP4_SCALE_E7(M) ((8 + (M)) * 0x1p-4f) +#define ROCMFP4_SCALE_E8(M) ((8 + (M)) * 0x1p-3f) +#define ROCMFP4_SCALE_E9(M) ((8 + (M)) * 0x1p-2f) +#define ROCMFP4_SCALE_E10(M) ((8 + (M)) * 0x1p-1f) +#define ROCMFP4_SCALE_E11(M) ((8 + (M)) * 0x1p0f) +#define ROCMFP4_SCALE_E12(M) ((8 + (M)) * 0x1p1f) +#define ROCMFP4_SCALE_E13(M) ((8 + (M)) * 0x1p2f) +#define ROCMFP4_SCALE_E14(M) ((8 + (M)) * 0x1p3f) +#define ROCMFP4_SCALE_E15(M) ((8 + (M)) * 0x1p4f) + +static __device__ __constant__ const float rocmfp4_scale_ue4m3_half_lut[127] = { + ROCMFP4_SCALE_SUB(0), ROCMFP4_SCALE_SUB(1), ROCMFP4_SCALE_SUB(2), ROCMFP4_SCALE_SUB(3), + ROCMFP4_SCALE_SUB(4), ROCMFP4_SCALE_SUB(5), ROCMFP4_SCALE_SUB(6), ROCMFP4_SCALE_SUB(7), + ROCMFP4_SCALE_E1(0), ROCMFP4_SCALE_E1(1), ROCMFP4_SCALE_E1(2), ROCMFP4_SCALE_E1(3), + ROCMFP4_SCALE_E1(4), ROCMFP4_SCALE_E1(5), ROCMFP4_SCALE_E1(6), ROCMFP4_SCALE_E1(7), + ROCMFP4_SCALE_E2(0), ROCMFP4_SCALE_E2(1), ROCMFP4_SCALE_E2(2), ROCMFP4_SCALE_E2(3), + ROCMFP4_SCALE_E2(4), ROCMFP4_SCALE_E2(5), ROCMFP4_SCALE_E2(6), ROCMFP4_SCALE_E2(7), + ROCMFP4_SCALE_E3(0), ROCMFP4_SCALE_E3(1), ROCMFP4_SCALE_E3(2), ROCMFP4_SCALE_E3(3), + ROCMFP4_SCALE_E3(4), ROCMFP4_SCALE_E3(5), ROCMFP4_SCALE_E3(6), ROCMFP4_SCALE_E3(7), + ROCMFP4_SCALE_E4(0), ROCMFP4_SCALE_E4(1), ROCMFP4_SCALE_E4(2), ROCMFP4_SCALE_E4(3), + ROCMFP4_SCALE_E4(4), ROCMFP4_SCALE_E4(5), ROCMFP4_SCALE_E4(6), ROCMFP4_SCALE_E4(7), + ROCMFP4_SCALE_E5(0), ROCMFP4_SCALE_E5(1), ROCMFP4_SCALE_E5(2), ROCMFP4_SCALE_E5(3), + ROCMFP4_SCALE_E5(4), ROCMFP4_SCALE_E5(5), ROCMFP4_SCALE_E5(6), ROCMFP4_SCALE_E5(7), + ROCMFP4_SCALE_E6(0), ROCMFP4_SCALE_E6(1), ROCMFP4_SCALE_E6(2), ROCMFP4_SCALE_E6(3), + ROCMFP4_SCALE_E6(4), ROCMFP4_SCALE_E6(5), ROCMFP4_SCALE_E6(6), ROCMFP4_SCALE_E6(7), + ROCMFP4_SCALE_E7(0), ROCMFP4_SCALE_E7(1), ROCMFP4_SCALE_E7(2), ROCMFP4_SCALE_E7(3), + ROCMFP4_SCALE_E7(4), ROCMFP4_SCALE_E7(5), ROCMFP4_SCALE_E7(6), ROCMFP4_SCALE_E7(7), + ROCMFP4_SCALE_E8(0), ROCMFP4_SCALE_E8(1), ROCMFP4_SCALE_E8(2), ROCMFP4_SCALE_E8(3), + ROCMFP4_SCALE_E8(4), ROCMFP4_SCALE_E8(5), ROCMFP4_SCALE_E8(6), ROCMFP4_SCALE_E8(7), + ROCMFP4_SCALE_E9(0), ROCMFP4_SCALE_E9(1), ROCMFP4_SCALE_E9(2), ROCMFP4_SCALE_E9(3), + ROCMFP4_SCALE_E9(4), ROCMFP4_SCALE_E9(5), ROCMFP4_SCALE_E9(6), ROCMFP4_SCALE_E9(7), + ROCMFP4_SCALE_E10(0), ROCMFP4_SCALE_E10(1), ROCMFP4_SCALE_E10(2), ROCMFP4_SCALE_E10(3), + ROCMFP4_SCALE_E10(4), ROCMFP4_SCALE_E10(5), ROCMFP4_SCALE_E10(6), ROCMFP4_SCALE_E10(7), + ROCMFP4_SCALE_E11(0), ROCMFP4_SCALE_E11(1), ROCMFP4_SCALE_E11(2), ROCMFP4_SCALE_E11(3), + ROCMFP4_SCALE_E11(4), ROCMFP4_SCALE_E11(5), ROCMFP4_SCALE_E11(6), ROCMFP4_SCALE_E11(7), + ROCMFP4_SCALE_E12(0), ROCMFP4_SCALE_E12(1), ROCMFP4_SCALE_E12(2), ROCMFP4_SCALE_E12(3), + ROCMFP4_SCALE_E12(4), ROCMFP4_SCALE_E12(5), ROCMFP4_SCALE_E12(6), ROCMFP4_SCALE_E12(7), + ROCMFP4_SCALE_E13(0), ROCMFP4_SCALE_E13(1), ROCMFP4_SCALE_E13(2), ROCMFP4_SCALE_E13(3), + ROCMFP4_SCALE_E13(4), ROCMFP4_SCALE_E13(5), ROCMFP4_SCALE_E13(6), ROCMFP4_SCALE_E13(7), + ROCMFP4_SCALE_E14(0), ROCMFP4_SCALE_E14(1), ROCMFP4_SCALE_E14(2), ROCMFP4_SCALE_E14(3), + ROCMFP4_SCALE_E14(4), ROCMFP4_SCALE_E14(5), ROCMFP4_SCALE_E14(6), ROCMFP4_SCALE_E14(7), + ROCMFP4_SCALE_E15(0), ROCMFP4_SCALE_E15(1), ROCMFP4_SCALE_E15(2), ROCMFP4_SCALE_E15(3), + ROCMFP4_SCALE_E15(4), ROCMFP4_SCALE_E15(5), ROCMFP4_SCALE_E15(6), +}; + +#undef ROCMFP4_SCALE_SUB +#undef ROCMFP4_SCALE_E1 +#undef ROCMFP4_SCALE_E2 +#undef ROCMFP4_SCALE_E3 +#undef ROCMFP4_SCALE_E4 +#undef ROCMFP4_SCALE_E5 +#undef ROCMFP4_SCALE_E6 +#undef ROCMFP4_SCALE_E7 +#undef ROCMFP4_SCALE_E8 +#undef ROCMFP4_SCALE_E9 +#undef ROCMFP4_SCALE_E10 +#undef ROCMFP4_SCALE_E11 +#undef ROCMFP4_SCALE_E12 +#undef ROCMFP4_SCALE_E13 +#undef ROCMFP4_SCALE_E14 +#undef ROCMFP4_SCALE_E15 +#endif + +static __device__ __forceinline__ float rocmfp4_u32_as_f32(uint32_t bits) { +#if defined(GGML_USE_HIP) + return __uint_as_float(bits); +#else + float result; + memcpy(&result, &bits, sizeof(float)); + return result; +#endif +} + +// ROCmFP4 validates scale bytes before backend execution, so HIP/ROCm hot +// paths can decode finite unsigned E4M3 half-scales directly without the +// generic FP8 NaN handling used by other formats. +static __device__ __forceinline__ float rocmfp4_ue4m3_to_fp32_half_finite(uint8_t x) { +#if defined(GGML_USE_HIP) && GGML_ROCMFP4_USE_SCALE_LUT + return x <= 0x7e ? rocmfp4_scale_ue4m3_half_lut[x] : 0.0f; +#else + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + + if (exp == 0) { + return (float) man * (1.0f / 1024.0f); + } + + const uint32_t bits = ((uint32_t) exp + 119u) << 23 | ((uint32_t) man << 20); + return rocmfp4_u32_as_f32(bits); +#endif +} + +static __device__ __forceinline__ float rocmfpx_ue4m3_to_fp32_finite(uint8_t x) { + if (x > 0x7e) { + return 0.0f; + } + + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + + if (exp == 0) { + return (float) man * (1.0f / 1024.0f); + } + + const uint32_t bits = ((uint32_t) exp + 119u) << 23 | ((uint32_t) man << 20); + return rocmfp4_u32_as_f32(bits); +} + +static __device__ __forceinline__ uint8_t rocmfpx_nearest_scale_ue4m3_cuda(float target_scale) { + if (!(target_scale > 0.0f) || !isfinite(target_scale)) { + return 0; + } + + uint8_t lo = 1; + uint8_t hi = 0x7e; + while (lo < hi) { + const uint8_t mid = lo + (hi - lo) / 2; + if (rocmfpx_ue4m3_to_fp32_finite(mid) < target_scale) { + lo = mid + 1; + } else { + hi = mid; + } + } + + if (lo == 1) { + return 1; + } + + const float hi_scale = rocmfpx_ue4m3_to_fp32_finite(lo); + const float lo_scale = rocmfpx_ue4m3_to_fp32_finite((uint8_t) (lo - 1)); + return (target_scale - lo_scale <= hi_scale - target_scale) ? (uint8_t) (lo - 1) : lo; +} + +static __device__ __forceinline__ int8_t rocmfp4_decode_i8(uint8_t q) { + q &= 0x0f; + const int mag3 = q & 0x07; + const int mag = mag3 <= 4 ? mag3 : 2*mag3 - 4; + return (q & 0x08) ? -mag : mag; +} diff --git a/server/deps/llama.cpp/ggml/rocmfpx/README.md b/server/deps/llama.cpp/ggml/rocmfpx/README.md new file mode 100644 index 000000000..a92eb4658 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfpx/README.md @@ -0,0 +1,106 @@ +# ROCmFPx Experimental Formats + +This folder contains the reference layer for the proposed ROCmFP3, ROCmFP6, and +ROCmFP8 quantization family. It is intentionally separate from `ggml/rocmfp4/` +so the promoted ROCmFP4 GGUF formats and kernels are not affected while the new +layouts are evaluated. + +## ROCmFP4 Instructions To Preserve + +ROCmFPX is a sibling model-weight quant family, not a new K/V-only compression +scheme. The promoted ROCmFP4 implementation remains the template for how the +family should behave in llama.cpp: + +- Keep 32-weight blocks so CPU, HIP, and Vulkan kernels can reuse the same + Q4/Q8-style reduction shape and GGUF row-size assumptions. +- Use finite unsigned UE4M3 scale bytes only. `0x7f` and sign-bit scale bytes + are invalid, matching the ROCmFP4 validation rule. +- Prefer reconstruction-MSE scale selection over plain max-abs scaling. ROCmFP4 + searches each 16-weight half-block; ROCmFP3 and ROCmFP6 follow the same + half-block policy, while ROCmFP8 currently uses one full-block scale. +- Preserve the ROCmFP4 kernel contract: CPU reference quant/dequant/dot first, + then HIP/Vulkan `CPY`, `GET_ROWS`, `SET_ROWS`, `MUL_MAT`, and `MUL_MAT_ID` + paths, with backend-op coverage before claiming runtime support. +- Keep dequant math explicit and deterministic: integer code times decoded + UE4M3 scale. ROCmFP4 uses the Codebook10 half-scale table; ROCmFPX formats + use their own integer code ranges but must retain the same finite-scale and + integer-dot discipline. + +The ROCmFP4 Codebook10 levels are not reused by FP3/FP6/FP8 directly: +`ROCmFP3` uses `0, +/-1, +/-2, +/-4`, `ROCmFP6` uses signed-magnitude levels up +to `31`, and `ROCmFP8` uses signed int8 levels clamped to `[-127, 127]`. +What is inherited is the block/scale/kernel/dequant contract. + +Current status (June 16, 2026): +- CPU reference quantize/dequantize exists for all three formats. +- `Q3_0_ROCMFPX`, `Q6_0_ROCMFPX`, and `Q8_0_ROCMFPX` are registered as + experimental GGUF tensor types. +- ROCm/HIP and Vulkan kernels support `CPY`, `GET_ROWS`, `SET_ROWS`, and + `MUL_MAT`/`MUL_MAT_ID` for all three formats. +- Qwen3-0.6B BF16 smoke tests pass on CPU, ROCm0, and Vulkan0. +- Default quant presets now include lean coherency routing: + - `Q3_0_ROCMFPX`: selective `Q5_K` on attention Q/O and early K/V, boosted + FFN-down at `Q5_K`, selective FFN-gate at `Q6_0_ROCMFPX`, bulk FFN-up on + `Q3_0_ROCMFPX`, embeddings/output at `Q4_0_ROCMFP4_FAST`. + - `Q6_0_ROCMFPX`: early attention and boosted FFN-down at `Q8_0_ROCMFPX`, + embeddings/output at `Q6_0_ROCMFPX`, bulk gate/up on `Q6_0_ROCMFPX`. + - `Q8_0_ROCMFPX`: pure FP8-family preset. +- Opt-in `*_AGENT` presets boost attention/FFN routing for tool-call / + Hermes / OpenClaw style workloads: + - `Q3_0_ROCMFPX_AGENT`, `Q6_0_ROCMFPX_AGENT`, `Q8_0_ROCMFPX_AGENT`. + - Routing is layered on top of LEAN; default presets are unchanged. +- FP3 and FP6 quantization use reconstruction-MSE scale selection per + 16-weight half-block. + +## Validation Script Index + +```text +scripts/check-rocmfpx-reference.sh # CPU reference math +scripts/check-rocmfpx-qwen-all.sh # core Qwen gates +scripts/check-rocmfpx-all.sh # qwen-all + optional smokes +scripts/check-rocmfpx-summary.sh # full JSON summary runner +scripts/sweep-rocmfpx-backend-ops.sh # test-backend-ops per backend +scripts/sweep-rocmfpx-agent-size-table.sh # LEAN vs AGENT MiB/BPW +scripts/sweep-rocmfpx-perplexity.sh # calibration PPL sweep +scripts/sweep-rocmfpx-decode-tune.sh # decode-tune matrix +scripts/build-rocmfpx-agent-fixtures.sh # proxy Hermes/OpenClaw AGENT GGUFs +``` + +## Layouts + +All formats use 32-weight blocks. + +| Format | Payload | Scale bytes | Block bytes | BPW | Purpose | +|---|---:|---:|---:|---:|---| +| `Q3_0_ROCMFPX` | 32 packed 3-bit codes | 2, one per 16 weights | 14 | 3.50 | Experimental low-bit candidate | +| `Q6_0_ROCMFPX` | 32 packed 6-bit codes | 2, one per 16 weights | 26 | 6.50 | Experimental quality candidate | +| `Q8_0_ROCMFPX` | 32 signed 8-bit codes | 1, one per 32 weights | 33 | 8.25 | Experimental high-quality reference | + +`ROCmFP3` uses a tiny signed codebook: `0, +/-1, +/-2, +/-4`. +`ROCmFP6` uses signed-magnitude integer levels up to `31`. +`ROCmFP8` uses signed int8 levels clamped to `[-127, 127]`. + +## Validation + +Reference math only: + +```bash +scripts/check-rocmfpx-reference.sh +``` + +Focused backend sweep from the experiment worktree: + +```bash +cmake --build build-strix-rocmfp4 --target test-backend-ops -j 8 +timeout 120 build-strix-rocmfp4/bin/test-backend-ops test -o MUL_MAT,GET_ROWS,CPY,SET_ROWS -b CPU +timeout 180 build-strix-rocmfp4/bin/test-backend-ops test -o MUL_MAT,GET_ROWS,CPY,SET_ROWS -b ROCm0 +timeout 180 build-strix-rocmfp4/bin/test-backend-ops test -o MUL_MAT,GET_ROWS,CPY,SET_ROWS -b Vulkan0 +``` + +Qwen3 BF16 coherency and decode-speed gates: + +```bash +MODEL=/home/caf/strix-fp4/models/rocmfpx-bf16-tests/Qwen3-0.6B-Q3_0_ROCMFPX_COHERENT-LEAN.gguf BACKEND=ROCm0 scripts/check-rocmfpx-qwen-coherency.sh +MODEL=/home/caf/strix-fp4/models/rocmfpx-bf16-tests/Qwen3-0.6B-Q3_0_ROCMFPX_COHERENT-LEAN.gguf BACKEND=ROCm0 scripts/check-rocmfpx-qwen-bench.sh +MODEL=/home/caf/strix-fp4/models/rocmfpx-bf16-tests/Qwen3-0.6B-Q3_0_ROCMFPX_COHERENT-LEAN.gguf BACKEND=ROCm0 scripts/check-rocmfpx-qwen-strict-json.sh +``` diff --git a/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c new file mode 100644 index 000000000..65dbca726 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c @@ -0,0 +1,1140 @@ +#include "rocmfpx.h" + +#include +#include +#include + +// Finite unsigned E4M3 scale bytes decoded to FP32. Precomputed from the same +// exp/mant formula rocmfpx_ue4m3_to_fp32() used to evaluate with ldexpf(): +// exp == 0 -> mant * 2^-10 ; otherwise (8 + mant) * 2^(exp - 11). +// The scale search re-decodes candidate bytes for every block, and dequant +// decodes a scale for every element, so keeping this as a table (identical to +// the former per-call ldexpf result) removes the transcendental from both hot +// paths without changing any produced value. +#define ROCMFPX_SCALE_SUB(M) ((M) * 0x1p-10f) +#define ROCMFPX_SCALE_E(B, M) ((8 + (M)) * (B)) + +static const float rocmfpx_scale_ue4m3[127] = { + ROCMFPX_SCALE_SUB(0), ROCMFPX_SCALE_SUB(1), ROCMFPX_SCALE_SUB(2), ROCMFPX_SCALE_SUB(3), + ROCMFPX_SCALE_SUB(4), ROCMFPX_SCALE_SUB(5), ROCMFPX_SCALE_SUB(6), ROCMFPX_SCALE_SUB(7), + ROCMFPX_SCALE_E(0x1p-10f,0), ROCMFPX_SCALE_E(0x1p-10f,1), ROCMFPX_SCALE_E(0x1p-10f,2), ROCMFPX_SCALE_E(0x1p-10f,3), + ROCMFPX_SCALE_E(0x1p-10f,4), ROCMFPX_SCALE_E(0x1p-10f,5), ROCMFPX_SCALE_E(0x1p-10f,6), ROCMFPX_SCALE_E(0x1p-10f,7), + ROCMFPX_SCALE_E(0x1p-9f,0), ROCMFPX_SCALE_E(0x1p-9f,1), ROCMFPX_SCALE_E(0x1p-9f,2), ROCMFPX_SCALE_E(0x1p-9f,3), + ROCMFPX_SCALE_E(0x1p-9f,4), ROCMFPX_SCALE_E(0x1p-9f,5), ROCMFPX_SCALE_E(0x1p-9f,6), ROCMFPX_SCALE_E(0x1p-9f,7), + ROCMFPX_SCALE_E(0x1p-8f,0), ROCMFPX_SCALE_E(0x1p-8f,1), ROCMFPX_SCALE_E(0x1p-8f,2), ROCMFPX_SCALE_E(0x1p-8f,3), + ROCMFPX_SCALE_E(0x1p-8f,4), ROCMFPX_SCALE_E(0x1p-8f,5), ROCMFPX_SCALE_E(0x1p-8f,6), ROCMFPX_SCALE_E(0x1p-8f,7), + ROCMFPX_SCALE_E(0x1p-7f,0), ROCMFPX_SCALE_E(0x1p-7f,1), ROCMFPX_SCALE_E(0x1p-7f,2), ROCMFPX_SCALE_E(0x1p-7f,3), + ROCMFPX_SCALE_E(0x1p-7f,4), ROCMFPX_SCALE_E(0x1p-7f,5), ROCMFPX_SCALE_E(0x1p-7f,6), ROCMFPX_SCALE_E(0x1p-7f,7), + ROCMFPX_SCALE_E(0x1p-6f,0), ROCMFPX_SCALE_E(0x1p-6f,1), ROCMFPX_SCALE_E(0x1p-6f,2), ROCMFPX_SCALE_E(0x1p-6f,3), + ROCMFPX_SCALE_E(0x1p-6f,4), ROCMFPX_SCALE_E(0x1p-6f,5), ROCMFPX_SCALE_E(0x1p-6f,6), ROCMFPX_SCALE_E(0x1p-6f,7), + ROCMFPX_SCALE_E(0x1p-5f,0), ROCMFPX_SCALE_E(0x1p-5f,1), ROCMFPX_SCALE_E(0x1p-5f,2), ROCMFPX_SCALE_E(0x1p-5f,3), + ROCMFPX_SCALE_E(0x1p-5f,4), ROCMFPX_SCALE_E(0x1p-5f,5), ROCMFPX_SCALE_E(0x1p-5f,6), ROCMFPX_SCALE_E(0x1p-5f,7), + ROCMFPX_SCALE_E(0x1p-4f,0), ROCMFPX_SCALE_E(0x1p-4f,1), ROCMFPX_SCALE_E(0x1p-4f,2), ROCMFPX_SCALE_E(0x1p-4f,3), + ROCMFPX_SCALE_E(0x1p-4f,4), ROCMFPX_SCALE_E(0x1p-4f,5), ROCMFPX_SCALE_E(0x1p-4f,6), ROCMFPX_SCALE_E(0x1p-4f,7), + ROCMFPX_SCALE_E(0x1p-3f,0), ROCMFPX_SCALE_E(0x1p-3f,1), ROCMFPX_SCALE_E(0x1p-3f,2), ROCMFPX_SCALE_E(0x1p-3f,3), + ROCMFPX_SCALE_E(0x1p-3f,4), ROCMFPX_SCALE_E(0x1p-3f,5), ROCMFPX_SCALE_E(0x1p-3f,6), ROCMFPX_SCALE_E(0x1p-3f,7), + ROCMFPX_SCALE_E(0x1p-2f,0), ROCMFPX_SCALE_E(0x1p-2f,1), ROCMFPX_SCALE_E(0x1p-2f,2), ROCMFPX_SCALE_E(0x1p-2f,3), + ROCMFPX_SCALE_E(0x1p-2f,4), ROCMFPX_SCALE_E(0x1p-2f,5), ROCMFPX_SCALE_E(0x1p-2f,6), ROCMFPX_SCALE_E(0x1p-2f,7), + ROCMFPX_SCALE_E(0x1p-1f,0), ROCMFPX_SCALE_E(0x1p-1f,1), ROCMFPX_SCALE_E(0x1p-1f,2), ROCMFPX_SCALE_E(0x1p-1f,3), + ROCMFPX_SCALE_E(0x1p-1f,4), ROCMFPX_SCALE_E(0x1p-1f,5), ROCMFPX_SCALE_E(0x1p-1f,6), ROCMFPX_SCALE_E(0x1p-1f,7), + ROCMFPX_SCALE_E(0x1p0f,0), ROCMFPX_SCALE_E(0x1p0f,1), ROCMFPX_SCALE_E(0x1p0f,2), ROCMFPX_SCALE_E(0x1p0f,3), + ROCMFPX_SCALE_E(0x1p0f,4), ROCMFPX_SCALE_E(0x1p0f,5), ROCMFPX_SCALE_E(0x1p0f,6), ROCMFPX_SCALE_E(0x1p0f,7), + ROCMFPX_SCALE_E(0x1p1f,0), ROCMFPX_SCALE_E(0x1p1f,1), ROCMFPX_SCALE_E(0x1p1f,2), ROCMFPX_SCALE_E(0x1p1f,3), + ROCMFPX_SCALE_E(0x1p1f,4), ROCMFPX_SCALE_E(0x1p1f,5), ROCMFPX_SCALE_E(0x1p1f,6), ROCMFPX_SCALE_E(0x1p1f,7), + ROCMFPX_SCALE_E(0x1p2f,0), ROCMFPX_SCALE_E(0x1p2f,1), ROCMFPX_SCALE_E(0x1p2f,2), ROCMFPX_SCALE_E(0x1p2f,3), + ROCMFPX_SCALE_E(0x1p2f,4), ROCMFPX_SCALE_E(0x1p2f,5), ROCMFPX_SCALE_E(0x1p2f,6), ROCMFPX_SCALE_E(0x1p2f,7), + ROCMFPX_SCALE_E(0x1p3f,0), ROCMFPX_SCALE_E(0x1p3f,1), ROCMFPX_SCALE_E(0x1p3f,2), ROCMFPX_SCALE_E(0x1p3f,3), + ROCMFPX_SCALE_E(0x1p3f,4), ROCMFPX_SCALE_E(0x1p3f,5), ROCMFPX_SCALE_E(0x1p3f,6), ROCMFPX_SCALE_E(0x1p3f,7), + ROCMFPX_SCALE_E(0x1p4f,0), ROCMFPX_SCALE_E(0x1p4f,1), ROCMFPX_SCALE_E(0x1p4f,2), ROCMFPX_SCALE_E(0x1p4f,3), + ROCMFPX_SCALE_E(0x1p4f,4), ROCMFPX_SCALE_E(0x1p4f,5), ROCMFPX_SCALE_E(0x1p4f,6), +}; + +#undef ROCMFPX_SCALE_SUB +#undef ROCMFPX_SCALE_E + +float rocmfpx_ue4m3_to_fp32(uint8_t e) { + return rocmfpx_scale_is_valid(e) ? rocmfpx_scale_ue4m3[e] : 0.0f; +} + +bool rocmfpx_scale_is_valid(uint8_t e) { + return e <= 0x7e; +} + +size_t rocmfpx_row_size_fp2(int64_t k) { + assert(k % QK_ROCMFP2 == 0); + return (size_t) (k / QK_ROCMFP2) * sizeof(block_rocmfp2); +} + +size_t rocmfpx_row_size_fp3(int64_t k) { + assert(k % QK_ROCMFP3 == 0); + return (size_t) (k / QK_ROCMFP3) * sizeof(block_rocmfp3); +} + +size_t rocmfpx_row_size_fp6(int64_t k) { + assert(k % QK_ROCMFP6 == 0); + return (size_t) (k / QK_ROCMFP6) * sizeof(block_rocmfp6); +} + +size_t rocmfpx_row_size_fp8(int64_t k) { + assert(k % QK_ROCMFP8 == 0); + return (size_t) (k / QK_ROCMFP8) * sizeof(block_rocmfp8); +} + +static uint8_t rocmfpx_nearest_scale_ue4m3(float target) { + if (!(target > 0.0f) || !isfinite(target)) { + return 0; + } + + uint8_t best_e = 1; + float best_err = fabsf(rocmfpx_ue4m3_to_fp32(best_e) - target); + + for (int e = 2; e <= 0x7e; ++e) { + const float err = fabsf(rocmfpx_ue4m3_to_fp32((uint8_t) e) - target); + if (err < best_err) { + best_err = err; + best_e = (uint8_t) e; + } + } + + return best_e; +} + +static float rocmfpx_max_abs(const float * x, int n) { + float max_abs = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + + const float ax = fabsf(x[i]); + if (ax > max_abs) { + max_abs = ax; + } + } + + return max_abs; +} + +static void rocmfpx_prepare_mse_weights( + float * dst, const float * x, int n, const float * quant_weights, float sigma2, + float * max_abs, float * max_abs_weight) { + *max_abs = 0.0f; + *max_abs_weight = 0.0f; + + for (int i = 0; i < n; ++i) { + const float ax = fabsf(x[i]); + const float qw = quant_weights[i]; + const float weight = isfinite(qw) && qw > 0.0f && isfinite(x[i]) ? qw * sqrtf(sigma2 + x[i]*x[i]) : 0.0f; + + if (isfinite(x[i])) { + if (ax > *max_abs) { + *max_abs = ax; + *max_abs_weight = weight; + } else if (ax == *max_abs && weight > *max_abs_weight) { + *max_abs_weight = weight; + } + } + + // Match llama.cpp imatrix weighting style: calibration importance is + // scaled by row energy so large activations stay protected. + dst[i] = weight; + } +} + +static void rocmfpx_set_bits(uint8_t * dst, int bit_pos, int nbits, uint32_t code) { + for (int bit = 0; bit < nbits; ++bit) { + const int absolute_bit = bit_pos + bit; + const int byte_index = absolute_bit >> 3; + const int bit_index = absolute_bit & 7; + + if ((code >> bit) & 1u) { + dst[byte_index] |= (uint8_t) (1u << bit_index); + } + } +} + +static uint32_t rocmfpx_get_bits(const uint8_t * src, int bit_pos, int nbits) { + uint32_t code = 0; + + for (int bit = 0; bit < nbits; ++bit) { + const int absolute_bit = bit_pos + bit; + const int byte_index = absolute_bit >> 3; + const int bit_index = absolute_bit & 7; + + code |= (uint32_t) ((src[byte_index] >> bit_index) & 1u) << bit; + } + + return code; +} + +// Starting 2-bit ROCmFP2 codebook. Keep this single definition easy to tune. +// TODO: affine scale+min would likely improve quality, but it would break the +// ROCmFPx family's unsigned-UE4M3 scale contract; revisit for a v2 layout. +static const float kvalues_rocmfp2[4] = ROCMFP2_KVALUES_INIT; + +static uint8_t rocmfpx_quantize_fp2_code(float x, float inv_scale) { + if (!isfinite(x) || inv_scale <= 0.0f) { + return 1; + } + + const float q = x * inv_scale; + uint8_t best_code = 0; + float best_err = fabsf(q - kvalues_rocmfp2[0]); + + for (uint8_t code = 1; code < 4; ++code) { + const float err = fabsf(q - kvalues_rocmfp2[code]); + if (err < best_err) { + best_err = err; + best_code = code; + } + } + + return best_code; +} + +static inline float rocmfpx_fp2_decoded_mag(float x, float inv_scale) { + return kvalues_rocmfp2[rocmfpx_quantize_fp2_code(x, inv_scale)]; +} + +static float rocmfpx_fp2_block_mse_for_scale(const float * x, int n, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + + const float y = rocmfpx_fp2_decoded_mag(x[i], inv_scale) * scale; + const float d = x[i] - y; + + err += d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static float rocmfpx_fp2_block_weighted_mse_for_scale(const float * x, int n, const float * mse_weights, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + + const float y = rocmfpx_fp2_decoded_mag(x[i], inv_scale) * scale; + const float d = x[i] - y; + + err += mse_weights[i]*d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static uint8_t rocmfpx_choose_scale_fp2_mse_impl(const float * x, int n, const float * mse_weights, float max_abs, float max_abs_weight) { + const uint8_t start_e = rocmfpx_nearest_scale_ue4m3(max_abs / 2.0f); + uint8_t best_e = start_e; + float best_err = INFINITY; + bool lower_done = false; + + for (int delta = 0; delta <= 125; ++delta) { + const int e0 = (int) start_e - delta; + if (!lower_done && e0 >= 1 && e0 <= 126) { + const float scale = rocmfpx_ue4m3_to_fp32((uint8_t) e0); + const float clip_delta = max_abs - 2.0f*scale; + const float clip_err = mse_weights ? max_abs_weight*clip_delta*clip_delta : clip_delta*clip_delta; + if (clip_delta > 0.0f && clip_err > best_err) { + lower_done = true; + } else { + const float err = mse_weights ? + rocmfpx_fp2_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e0, best_err) : + rocmfpx_fp2_block_mse_for_scale(x, n, (uint8_t) e0, best_err); + if (err < best_err || (err == best_err && e0 < best_e)) { + best_err = err; + best_e = (uint8_t) e0; + } + } + } + + const int e1 = (int) start_e + delta; + if (delta != 0 && e1 >= 1 && e1 <= 126) { + const float err = mse_weights ? + rocmfpx_fp2_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e1, best_err) : + rocmfpx_fp2_block_mse_for_scale(x, n, (uint8_t) e1, best_err); + if (err < best_err || (err == best_err && e1 < best_e)) { + best_err = err; + best_e = (uint8_t) e1; + } + } + + if ((lower_done || e0 <= 1) && e1 >= 126) { + break; + } + } + + return best_e; +} + +static uint8_t rocmfpx_choose_scale_fp2_mse(const float * x, int n) { + const float max_abs = rocmfpx_max_abs(x, n); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfpx_choose_scale_fp2_mse_impl(x, n, NULL, max_abs, 0.0f); +} + +static uint8_t rocmfpx_choose_scale_fp2_weighted_mse(const float * x, int n, const float * quant_weights, float sigma2) { + assert(n <= QK_ROCMFP2); + float mse_weights[QK_ROCMFP2]; + float max_abs; + float max_abs_weight; + rocmfpx_prepare_mse_weights(mse_weights, x, n, quant_weights, sigma2, &max_abs, &max_abs_weight); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfpx_choose_scale_fp2_mse_impl(x, n, mse_weights, max_abs, max_abs_weight); +} + +static int rocmfpx_decode_fp3_code(uint8_t code) { + static const int mag[4] = { 0, 1, 2, 4 }; + const int value = mag[code & 3u]; + return (code & 4u) ? -value : value; +} + +static uint8_t rocmfpx_quantize_fp3_code(float x, float inv_scale) { + if (!isfinite(x) || inv_scale <= 0.0f) { + return 0; + } + + const float ax = fabsf(x * inv_scale); + uint8_t mag; + + if (ax <= 0.5f) { + mag = 0; + } else if (ax <= 1.5f) { + mag = 1; + } else if (ax <= 3.0f) { + mag = 2; + } else { + mag = 3; + } + + return mag == 0 ? 0 : (uint8_t) ((x < 0.0f ? 4u : 0u) | mag); +} + +// Fused threshold + decode used only inside the exhaustive scale search, which +// re-scans every element for every candidate scale byte. Returns the same +// signed decoded magnitude that +// rocmfpx_decode_fp3_code(rocmfpx_quantize_fp3_code(x, inv_scale)) produces +// (fp3 magnitudes {0,1,2,4}), so quantized output stays bit-identical. +static inline float rocmfpx_fp3_decoded_mag(float x, float inv_scale) { + const float a = fabsf(x * inv_scale); + float mag; + if (a <= 0.5f) { + return 0.0f; + } else if (a <= 1.5f) { + mag = 1.0f; + } else if (a <= 3.0f) { + mag = 2.0f; + } else { + mag = 4.0f; + } + return x < 0.0f ? -mag : mag; +} + +static float rocmfpx_fp3_block_mse_for_scale(const float * x, int n, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + + const float y = rocmfpx_fp3_decoded_mag(x[i], inv_scale) * scale; + const float d = x[i] - y; + + err += d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static float rocmfpx_fp3_block_weighted_mse_for_scale(const float * x, int n, const float * mse_weights, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + + const float y = rocmfpx_fp3_decoded_mag(x[i], inv_scale) * scale; + const float d = x[i] - y; + + err += mse_weights[i]*d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static uint8_t rocmfpx_choose_scale_fp3_mse_impl(const float * x, int n, const float * mse_weights, float max_abs, float max_abs_weight) { + const uint8_t start_e = rocmfpx_nearest_scale_ue4m3(max_abs / 4.0f); + uint8_t best_e = start_e; + float best_err = INFINITY; + bool lower_done = false; + + for (int delta = 0; delta <= 125; ++delta) { + const int e0 = (int) start_e - delta; + if (!lower_done && e0 >= 1 && e0 <= 126) { + const float scale = rocmfpx_ue4m3_to_fp32((uint8_t) e0); + const float clip_delta = max_abs - 4.0f*scale; + const float clip_err = mse_weights ? max_abs_weight*clip_delta*clip_delta : clip_delta*clip_delta; + if (clip_delta > 0.0f && clip_err > best_err) { + lower_done = true; + } else { + const float err = mse_weights ? + rocmfpx_fp3_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e0, best_err) : + rocmfpx_fp3_block_mse_for_scale(x, n, (uint8_t) e0, best_err); + if (err < best_err || (err == best_err && e0 < best_e)) { + best_err = err; + best_e = (uint8_t) e0; + } + } + } + + const int e1 = (int) start_e + delta; + if (delta != 0 && e1 >= 1 && e1 <= 126) { + const float err = mse_weights ? + rocmfpx_fp3_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e1, best_err) : + rocmfpx_fp3_block_mse_for_scale(x, n, (uint8_t) e1, best_err); + if (err < best_err || (err == best_err && e1 < best_e)) { + best_err = err; + best_e = (uint8_t) e1; + } + } + + if ((lower_done || e0 <= 1) && e1 >= 126) { + break; + } + } + + return best_e; +} + +static uint8_t rocmfpx_choose_scale_fp3_mse(const float * x, int n) { + const float max_abs = rocmfpx_max_abs(x, n); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfpx_choose_scale_fp3_mse_impl(x, n, NULL, max_abs, 0.0f); +} + +static uint8_t rocmfpx_choose_scale_fp3_weighted_mse(const float * x, int n, const float * quant_weights, float sigma2) { + assert(n <= QK_ROCMFP3); + float mse_weights[QK_ROCMFP3]; + float max_abs; + float max_abs_weight; + rocmfpx_prepare_mse_weights(mse_weights, x, n, quant_weights, sigma2, &max_abs, &max_abs_weight); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfpx_choose_scale_fp3_mse_impl(x, n, mse_weights, max_abs, max_abs_weight); +} + +static int rocmfpx_decode_fp6_code(uint8_t code) { + const int value = code & 31u; + return (code & 32u) ? -value : value; +} + +static uint8_t rocmfpx_quantize_fp6_code(float x, float inv_scale) { + if (!isfinite(x) || inv_scale <= 0.0f) { + return 0; + } + + int mag = (int) lroundf(fabsf(x * inv_scale)); + if (mag > 31) { + mag = 31; + } + + return mag == 0 ? 0 : (uint8_t) ((x < 0.0f ? 32u : 0u) | (uint8_t) mag); +} + +// Fused round + clamp + decode for the fp6 scale search. Returns the same signed +// decoded magnitude as rocmfpx_decode_fp6_code(rocmfpx_quantize_fp6_code(...)) +// (nearest integer in [0,31], signed), keeping quantized output bit-identical. +static inline float rocmfpx_fp6_decoded_mag(float x, float inv_scale) { + int mag = (int) lroundf(fabsf(x * inv_scale)); + if (mag > 31) { + mag = 31; + } + if (mag == 0) { + return 0.0f; + } + return x < 0.0f ? -(float) mag : (float) mag; +} + +static float rocmfpx_fp6_block_mse_for_scale(const float * x, int n, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + const float y = rocmfpx_fp6_decoded_mag(x[i], inv_scale) * scale; + const float d = x[i] - y; + err += d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static float rocmfpx_fp6_block_weighted_mse_for_scale(const float * x, int n, const float * mse_weights, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + const float y = rocmfpx_fp6_decoded_mag(x[i], inv_scale) * scale; + const float d = x[i] - y; + err += mse_weights[i]*d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static uint8_t rocmfpx_choose_scale_fp6_mse_impl(const float * x, int n, const float * mse_weights, float max_abs, float max_abs_weight) { + const uint8_t start_e = rocmfpx_nearest_scale_ue4m3(max_abs / 31.0f); + uint8_t best_e = start_e; + float best_err = INFINITY; + bool lower_done = false; + + for (int delta = 0; delta <= 125; ++delta) { + const int e0 = (int) start_e - delta; + if (!lower_done && e0 >= 1 && e0 <= 126) { + const float scale = rocmfpx_ue4m3_to_fp32((uint8_t) e0); + const float clip_delta = max_abs - 31.0f*scale; + const float clip_err = mse_weights ? max_abs_weight*clip_delta*clip_delta : clip_delta*clip_delta; + if (clip_delta > 0.0f && clip_err > best_err) { + lower_done = true; + } else { + const float err = mse_weights ? + rocmfpx_fp6_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e0, best_err) : + rocmfpx_fp6_block_mse_for_scale(x, n, (uint8_t) e0, best_err); + if (err < best_err || (err == best_err && e0 < best_e)) { + best_err = err; + best_e = (uint8_t) e0; + } + } + } + + const int e1 = (int) start_e + delta; + if (delta != 0 && e1 >= 1 && e1 <= 126) { + const float err = mse_weights ? + rocmfpx_fp6_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e1, best_err) : + rocmfpx_fp6_block_mse_for_scale(x, n, (uint8_t) e1, best_err); + if (err < best_err || (err == best_err && e1 < best_e)) { + best_err = err; + best_e = (uint8_t) e1; + } + } + + if ((lower_done || e0 <= 1) && e1 >= 126) { + break; + } + } + + return best_e; +} + +static uint8_t rocmfpx_choose_scale_fp6_mse(const float * x, int n) { + const float max_abs = rocmfpx_max_abs(x, n); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfpx_choose_scale_fp6_mse_impl(x, n, NULL, max_abs, 0.0f); +} + +static uint8_t rocmfpx_choose_scale_fp6_weighted_mse(const float * x, int n, const float * quant_weights, float sigma2) { + assert(n <= QK_ROCMFP6); + float mse_weights[QK_ROCMFP6]; + float max_abs; + float max_abs_weight; + rocmfpx_prepare_mse_weights(mse_weights, x, n, quant_weights, sigma2, &max_abs, &max_abs_weight); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + return rocmfpx_choose_scale_fp6_mse_impl(x, n, mse_weights, max_abs, max_abs_weight); +} + +static int8_t rocmfpx_quantize_fp8_code(float x, float inv_scale) { + if (!isfinite(x) || inv_scale <= 0.0f) { + return 0; + } + + int q = (int) lroundf(x * inv_scale); + if (q > 127) { + q = 127; + } else if (q < -127) { + q = -127; + } + + return (int8_t) q; +} + +static float rocmfpx_fp8_block_weighted_mse_for_scale(const float * x, int n, const float * mse_weights, uint8_t e, float best_err) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + if (!isfinite(x[i])) { + continue; + } + + const int8_t code = rocmfpx_quantize_fp8_code(x[i], inv_scale); + const float y = (float) code * scale; + const float d = x[i] - y; + + err += mse_weights[i]*d*d; + if (err > best_err) { + return err; + } + } + + return err; +} + +static uint8_t rocmfpx_choose_scale_fp8_weighted_mse(const float * x, int n, const float * quant_weights, float sigma2) { + assert(n <= QK_ROCMFP8); + float mse_weights[QK_ROCMFP8]; + float max_abs; + float max_abs_weight; + rocmfpx_prepare_mse_weights(mse_weights, x, n, quant_weights, sigma2, &max_abs, &max_abs_weight); + if (!(max_abs > 0.0f) || !isfinite(max_abs)) { + return 0; + } + + const uint8_t start_e = rocmfpx_nearest_scale_ue4m3(max_abs / 127.0f); + uint8_t best_e = start_e; + float best_err = INFINITY; + bool lower_done = false; + + for (int delta = 0; delta <= 125; ++delta) { + const int e0 = (int) start_e - delta; + if (!lower_done && e0 >= 1 && e0 <= 126) { + const float scale = rocmfpx_ue4m3_to_fp32((uint8_t) e0); + const float clip_delta = max_abs - 127.0f*scale; + if (clip_delta > 0.0f && max_abs_weight*clip_delta*clip_delta > best_err) { + lower_done = true; + } else { + const float err = rocmfpx_fp8_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e0, best_err); + if (err < best_err || (err == best_err && e0 < best_e)) { + best_err = err; + best_e = (uint8_t) e0; + } + } + } + + const int e1 = (int) start_e + delta; + if (delta != 0 && e1 >= 1 && e1 <= 126) { + const float err = rocmfpx_fp8_block_weighted_mse_for_scale(x, n, mse_weights, (uint8_t) e1, best_err); + if (err < best_err || (err == best_err && e1 < best_e)) { + best_err = err; + best_e = (uint8_t) e1; + } + } + + if ((lower_done || e0 <= 1) && e1 >= 126) { + break; + } + } + + return best_e; +} + +void rocmfpx_quantize_row_fp2_ref(const float * GGML_RESTRICT x, block_rocmfp2 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP2 == 0); + + const int64_t nb = k / QK_ROCMFP2; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP2; + block_rocmfp2 * yb = y + ib; + + memset(yb->qs, 0, sizeof(yb->qs)); + + for (int half = 0; half < 2; ++half) { + const float * xh = xb + half*(QK_ROCMFP2/2); + yb->e[half] = rocmfpx_choose_scale_fp2_mse(xh, QK_ROCMFP2/2); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e[half]); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int j = 0; j < QK_ROCMFP2/2; ++j) { + const int i = half*(QK_ROCMFP2/2) + j; + const uint8_t code = rocmfpx_quantize_fp2_code(xb[i], inv_scale); + yb->qs[i >> 2] |= (uint8_t) (code << (2*(i & 3))); + } + } + } +} + +static void rocmfpx_quantize_row_fp2_weighted( + const float * GGML_RESTRICT x, block_rocmfp2 * GGML_RESTRICT y, int64_t k, const float * GGML_RESTRICT quant_weights) { + assert(k % QK_ROCMFP2 == 0); + + float sum_x2 = 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum_x2 += isfinite(x[i]) ? x[i]*x[i] : 0.0f; + } + const float sigma2 = sum_x2 / (float) k; + + const int64_t nb = k / QK_ROCMFP2; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP2; + const float * qw = quant_weights ? quant_weights + ib*QK_ROCMFP2 : NULL; + block_rocmfp2 * yb = y + ib; + + memset(yb->qs, 0, sizeof(yb->qs)); + + for (int half = 0; half < 2; ++half) { + const int half_off = half*(QK_ROCMFP2/2); + const float * xh = xb + half_off; + const float * qh = qw ? qw + half_off : NULL; + yb->e[half] = qh ? + rocmfpx_choose_scale_fp2_weighted_mse(xh, QK_ROCMFP2/2, qh, sigma2) : + rocmfpx_choose_scale_fp2_mse(xh, QK_ROCMFP2/2); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e[half]); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int j = 0; j < QK_ROCMFP2/2; ++j) { + const int i = half_off + j; + const uint8_t code = rocmfpx_quantize_fp2_code(xb[i], inv_scale); + yb->qs[i >> 2] |= (uint8_t) (code << (2*(i & 3))); + } + } + } +} + +void rocmfpx_dequantize_row_fp2(const block_rocmfp2 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP2 == 0); + + const int64_t nb = k / QK_ROCMFP2; + for (int64_t ib = 0; ib < nb; ++ib) { + const block_rocmfp2 * xb = x + ib; + float * yb = y + ib*QK_ROCMFP2; + + for (int i = 0; i < QK_ROCMFP2; ++i) { + const float scale = rocmfpx_ue4m3_to_fp32(xb->e[i >= QK_ROCMFP2/2]); + const uint8_t code = (uint8_t) ((xb->qs[i >> 2] >> (2*(i & 3))) & 3u); + yb[i] = kvalues_rocmfp2[code] * scale; + } + } +} + +void rocmfpx_quantize_row_fp2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + rocmfpx_quantize_row_fp2_ref(x, (block_rocmfp2 *) y, k); +} + +size_t rocmfpx_quantize_fp2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { + const size_t row_size = rocmfpx_row_size_fp2(n_per_row); + char * qrow = (char *) dst; + + for (int64_t row = 0; row < nrows; ++row) { + if (imatrix) { + rocmfpx_quantize_row_fp2_weighted(src + row*n_per_row, (block_rocmfp2 *) qrow, n_per_row, imatrix); + } else { + rocmfpx_quantize_row_fp2_ref(src + row*n_per_row, (block_rocmfp2 *) qrow, n_per_row); + } + qrow += row_size; + } + + return (size_t) nrows * row_size; +} + +void rocmfpx_quantize_row_fp3_ref(const float * GGML_RESTRICT x, block_rocmfp3 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP3 == 0); + + const int64_t nb = k / QK_ROCMFP3; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP3; + block_rocmfp3 * yb = y + ib; + + memset(yb->qs, 0, sizeof(yb->qs)); + + for (int half = 0; half < 2; ++half) { + const float * xh = xb + half*(QK_ROCMFP3/2); + yb->e[half] = rocmfpx_choose_scale_fp3_mse(xh, QK_ROCMFP3/2); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e[half]); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int j = 0; j < QK_ROCMFP3/2; ++j) { + const int i = half*(QK_ROCMFP3/2) + j; + const uint8_t code = rocmfpx_quantize_fp3_code(xb[i], inv_scale); + rocmfpx_set_bits(yb->qs, i*3, 3, code); + } + } + } +} + +static void rocmfpx_quantize_row_fp3_weighted( + const float * GGML_RESTRICT x, block_rocmfp3 * GGML_RESTRICT y, int64_t k, const float * GGML_RESTRICT quant_weights) { + assert(k % QK_ROCMFP3 == 0); + + float sum_x2 = 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum_x2 += isfinite(x[i]) ? x[i]*x[i] : 0.0f; + } + const float sigma2 = sum_x2 / (float) k; + + const int64_t nb = k / QK_ROCMFP3; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP3; + const float * qw = quant_weights ? quant_weights + ib*QK_ROCMFP3 : NULL; + block_rocmfp3 * yb = y + ib; + + memset(yb->qs, 0, sizeof(yb->qs)); + + for (int half = 0; half < 2; ++half) { + const int half_off = half*(QK_ROCMFP3/2); + const float * xh = xb + half_off; + const float * qh = qw ? qw + half_off : NULL; + yb->e[half] = qh ? + rocmfpx_choose_scale_fp3_weighted_mse(xh, QK_ROCMFP3/2, qh, sigma2) : + rocmfpx_choose_scale_fp3_mse(xh, QK_ROCMFP3/2); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e[half]); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int j = 0; j < QK_ROCMFP3/2; ++j) { + const int i = half_off + j; + const uint8_t code = rocmfpx_quantize_fp3_code(xb[i], inv_scale); + rocmfpx_set_bits(yb->qs, i*3, 3, code); + } + } + } +} + +void rocmfpx_dequantize_row_fp3(const block_rocmfp3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP3 == 0); + + const int64_t nb = k / QK_ROCMFP3; + for (int64_t ib = 0; ib < nb; ++ib) { + const block_rocmfp3 * xb = x + ib; + float * yb = y + ib*QK_ROCMFP3; + + for (int i = 0; i < QK_ROCMFP3; ++i) { + const float scale = rocmfpx_ue4m3_to_fp32(xb->e[i >= QK_ROCMFP3/2]); + const uint8_t code = (uint8_t) rocmfpx_get_bits(xb->qs, i*3, 3); + yb[i] = (float) rocmfpx_decode_fp3_code(code) * scale; + } + } +} + +void rocmfpx_quantize_row_fp3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + rocmfpx_quantize_row_fp3_ref(x, (block_rocmfp3 *) y, k); +} + +size_t rocmfpx_quantize_fp3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { + const size_t row_size = rocmfpx_row_size_fp3(n_per_row); + char * qrow = (char *) dst; + + for (int64_t row = 0; row < nrows; ++row) { + if (imatrix) { + rocmfpx_quantize_row_fp3_weighted(src + row*n_per_row, (block_rocmfp3 *) qrow, n_per_row, imatrix); + } else { + rocmfpx_quantize_row_fp3_ref(src + row*n_per_row, (block_rocmfp3 *) qrow, n_per_row); + } + qrow += row_size; + } + + return (size_t) nrows * row_size; +} + +static void rocmfpx_quantize_row_fp6_weighted( + const float * GGML_RESTRICT x, block_rocmfp6 * GGML_RESTRICT y, int64_t k, const float * GGML_RESTRICT quant_weights) { + assert(k % QK_ROCMFP6 == 0); + + float sum_x2 = 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum_x2 += isfinite(x[i]) ? x[i]*x[i] : 0.0f; + } + const float sigma2 = sum_x2 / (float) k; + + const int64_t nb = k / QK_ROCMFP6; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP6; + const float * qw = quant_weights ? quant_weights + ib*QK_ROCMFP6 : NULL; + block_rocmfp6 * yb = y + ib; + + memset(yb->qs, 0, sizeof(yb->qs)); + + for (int half = 0; half < 2; ++half) { + const int half_off = half*(QK_ROCMFP6/2); + const float * xh = xb + half_off; + const float * qh = qw ? qw + half_off : NULL; + yb->e[half] = qh ? + rocmfpx_choose_scale_fp6_weighted_mse(xh, QK_ROCMFP6/2, qh, sigma2) : + rocmfpx_choose_scale_fp6_mse(xh, QK_ROCMFP6/2); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e[half]); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int j = 0; j < QK_ROCMFP6/2; ++j) { + const int i = half_off + j; + const uint8_t code = rocmfpx_quantize_fp6_code(xb[i], inv_scale); + rocmfpx_set_bits(yb->qs, i*6, 6, code); + } + } + } +} + +void rocmfpx_quantize_row_fp6_ref(const float * GGML_RESTRICT x, block_rocmfp6 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP6 == 0); + + const int64_t nb = k / QK_ROCMFP6; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP6; + block_rocmfp6 * yb = y + ib; + + memset(yb->qs, 0, sizeof(yb->qs)); + + for (int half = 0; half < 2; ++half) { + const float * xh = xb + half*(QK_ROCMFP6/2); + yb->e[half] = rocmfpx_choose_scale_fp6_mse(xh, QK_ROCMFP6/2); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e[half]); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int j = 0; j < QK_ROCMFP6/2; ++j) { + const int i = half*(QK_ROCMFP6/2) + j; + const uint8_t code = rocmfpx_quantize_fp6_code(xb[i], inv_scale); + rocmfpx_set_bits(yb->qs, i*6, 6, code); + } + } + } +} + +void rocmfpx_dequantize_row_fp6(const block_rocmfp6 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP6 == 0); + + const int64_t nb = k / QK_ROCMFP6; + for (int64_t ib = 0; ib < nb; ++ib) { + const block_rocmfp6 * xb = x + ib; + float * yb = y + ib*QK_ROCMFP6; + + for (int i = 0; i < QK_ROCMFP6; ++i) { + const float scale = rocmfpx_ue4m3_to_fp32(xb->e[i >= QK_ROCMFP6/2]); + const uint8_t code = (uint8_t) rocmfpx_get_bits(xb->qs, i*6, 6); + yb[i] = (float) rocmfpx_decode_fp6_code(code) * scale; + } + } +} + +void rocmfpx_quantize_row_fp6(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + rocmfpx_quantize_row_fp6_ref(x, (block_rocmfp6 *) y, k); +} + +size_t rocmfpx_quantize_fp6(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { + const size_t row_size = rocmfpx_row_size_fp6(n_per_row); + char * qrow = (char *) dst; + + for (int64_t row = 0; row < nrows; ++row) { + if (imatrix) { + rocmfpx_quantize_row_fp6_weighted(src + row*n_per_row, (block_rocmfp6 *) qrow, n_per_row, imatrix); + } else { + rocmfpx_quantize_row_fp6_ref(src + row*n_per_row, (block_rocmfp6 *) qrow, n_per_row); + } + qrow += row_size; + } + + return (size_t) nrows * row_size; +} + +static void rocmfpx_quantize_row_fp8_weighted( + const float * GGML_RESTRICT x, block_rocmfp8 * GGML_RESTRICT y, int64_t k, const float * GGML_RESTRICT quant_weights) { + assert(k % QK_ROCMFP8 == 0); + + float sum_x2 = 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum_x2 += isfinite(x[i]) ? x[i]*x[i] : 0.0f; + } + const float sigma2 = sum_x2 / (float) k; + + const int64_t nb = k / QK_ROCMFP8; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP8; + const float * qw = quant_weights ? quant_weights + ib*QK_ROCMFP8 : NULL; + block_rocmfp8 * yb = y + ib; + + yb->e = qw ? rocmfpx_choose_scale_fp8_weighted_mse(xb, QK_ROCMFP8, qw, sigma2) : + rocmfpx_nearest_scale_ue4m3(rocmfpx_max_abs(xb, QK_ROCMFP8) / 127.0f); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int i = 0; i < QK_ROCMFP8; ++i) { + yb->qs[i] = rocmfpx_quantize_fp8_code(xb[i], inv_scale); + } + } +} + +void rocmfpx_quantize_row_fp8_ref(const float * GGML_RESTRICT x, block_rocmfp8 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP8 == 0); + + const int64_t nb = k / QK_ROCMFP8; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP8; + block_rocmfp8 * yb = y + ib; + + const float max_abs = rocmfpx_max_abs(xb, QK_ROCMFP8); + yb->e = rocmfpx_nearest_scale_ue4m3(max_abs / 127.0f); + + const float scale = rocmfpx_ue4m3_to_fp32(yb->e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + + for (int i = 0; i < QK_ROCMFP8; ++i) { + yb->qs[i] = rocmfpx_quantize_fp8_code(xb[i], inv_scale); + } + } +} + +void rocmfpx_dequantize_row_fp8(const block_rocmfp8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_ROCMFP8 == 0); + + const int64_t nb = k / QK_ROCMFP8; + for (int64_t ib = 0; ib < nb; ++ib) { + const block_rocmfp8 * xb = x + ib; + float * yb = y + ib*QK_ROCMFP8; + + const float scale = rocmfpx_ue4m3_to_fp32(xb->e); + for (int i = 0; i < QK_ROCMFP8; ++i) { + yb[i] = (float) xb->qs[i] * scale; + } + } +} + +void rocmfpx_quantize_row_fp8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + rocmfpx_quantize_row_fp8_ref(x, (block_rocmfp8 *) y, k); +} + +size_t rocmfpx_quantize_fp8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { + const size_t row_size = rocmfpx_row_size_fp8(n_per_row); + char * qrow = (char *) dst; + + for (int64_t row = 0; row < nrows; ++row) { + if (imatrix) { + rocmfpx_quantize_row_fp8_weighted(src + row*n_per_row, (block_rocmfp8 *) qrow, n_per_row, imatrix); + } else { + rocmfpx_quantize_row_fp8_ref(src + row*n_per_row, (block_rocmfp8 *) qrow, n_per_row); + } + qrow += row_size; + } + + return (size_t) nrows * row_size; +} + +bool rocmfpx_validate_row_data_fp2(const void * data, size_t nbytes) { + if (nbytes % sizeof(block_rocmfp2) != 0) { + return false; + } + + const block_rocmfp2 * blocks = (const block_rocmfp2 *) data; + const size_t nb = nbytes / sizeof(block_rocmfp2); + + for (size_t i = 0; i < nb; ++i) { + if (!rocmfpx_scale_is_valid(blocks[i].e[0]) || !rocmfpx_scale_is_valid(blocks[i].e[1])) { + return false; + } + } + + return true; +} + +bool rocmfpx_validate_row_data_fp3(const void * data, size_t nbytes) { + if (nbytes % sizeof(block_rocmfp3) != 0) { + return false; + } + + const block_rocmfp3 * blocks = (const block_rocmfp3 *) data; + const size_t nb = nbytes / sizeof(block_rocmfp3); + + for (size_t i = 0; i < nb; ++i) { + if (!rocmfpx_scale_is_valid(blocks[i].e[0]) || !rocmfpx_scale_is_valid(blocks[i].e[1])) { + return false; + } + } + + return true; +} + +bool rocmfpx_validate_row_data_fp6(const void * data, size_t nbytes) { + if (nbytes % sizeof(block_rocmfp6) != 0) { + return false; + } + + const block_rocmfp6 * blocks = (const block_rocmfp6 *) data; + const size_t nb = nbytes / sizeof(block_rocmfp6); + + for (size_t i = 0; i < nb; ++i) { + if (!rocmfpx_scale_is_valid(blocks[i].e[0]) || !rocmfpx_scale_is_valid(blocks[i].e[1])) { + return false; + } + } + + return true; +} + +bool rocmfpx_validate_row_data_fp8(const void * data, size_t nbytes) { + if (nbytes % sizeof(block_rocmfp8) != 0) { + return false; + } + + const block_rocmfp8 * blocks = (const block_rocmfp8 *) data; + const size_t nb = nbytes / sizeof(block_rocmfp8); + + for (size_t i = 0; i < nb; ++i) { + if (!rocmfpx_scale_is_valid(blocks[i].e)) { + return false; + } + } + + return true; +} diff --git a/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h new file mode 100644 index 000000000..f9d6cb896 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define QK_ROCMFPX 32 + +#define QK_ROCMFP2 QK_ROCMFPX +#define QK_ROCMFP3 QK_ROCMFPX +#define QK_ROCMFP6 QK_ROCMFPX +#define QK_ROCMFP8 QK_ROCMFPX + +#define QS_ROCMFP2 ((QK_ROCMFP2 * 2) / 8) +#define QS_ROCMFP3 ((QK_ROCMFP3 * 3) / 8) +#define QS_ROCMFP6 ((QK_ROCMFP6 * 6) / 8) +#define QS_ROCMFP8 QK_ROCMFP8 + +#define QR_ROCMFP2 1 +#define QI_ROCMFP2 (QS_ROCMFP2 / 4) // 8B qs = 2 int32; decoupled from QR_ROCMFP2 (QR stays 1 for convert.cu dequant) + +#define ROCMFP2_KVALUE_0_I8 (-1) +#define ROCMFP2_KVALUE_1_I8 0 +#define ROCMFP2_KVALUE_2_I8 1 +#define ROCMFP2_KVALUE_3_I8 2 +#define ROCMFP2_KVALUES_INIT { (float) ROCMFP2_KVALUE_0_I8, (float) ROCMFP2_KVALUE_1_I8, (float) ROCMFP2_KVALUE_2_I8, (float) ROCMFP2_KVALUE_3_I8 } + +#define QR_ROCMFP3 1 +#define QI_ROCMFP3 (QK_ROCMFP3 / (4 * QR_ROCMFP3)) + +#define QR_ROCMFP6 1 +#define QI_ROCMFP6 (QK_ROCMFP6 / (4 * QR_ROCMFP6)) + +#define QR_ROCMFP8 1 +#define QI_ROCMFP8 (QK_ROCMFP8 / (4 * QR_ROCMFP8)) + +// AMD-native experimental family layouts. The GGUF types are registered, but +// the layouts stay isolated from the promoted ROCmFP4 formats while evaluated. +typedef struct { + uint8_t qs[QS_ROCMFP2]; + uint8_t e[2]; +} block_rocmfp2; + +typedef struct { + uint8_t qs[QS_ROCMFP3]; + uint8_t e[2]; +} block_rocmfp3; + +typedef struct { + uint8_t qs[QS_ROCMFP6]; + uint8_t e[2]; +} block_rocmfp6; + +typedef struct { + int8_t qs[QS_ROCMFP8]; + uint8_t e; +} block_rocmfp8; + +#if defined(__cplusplus) +static_assert(sizeof(block_rocmfp2) == QS_ROCMFP2 + 2*sizeof(uint8_t), "wrong rocmfp2 block size/padding"); +static_assert(sizeof(block_rocmfp3) == QS_ROCMFP3 + 2*sizeof(uint8_t), "wrong rocmfp3 block size/padding"); +static_assert(sizeof(block_rocmfp6) == QS_ROCMFP6 + 2*sizeof(uint8_t), "wrong rocmfp6 block size/padding"); +static_assert(sizeof(block_rocmfp8) == QS_ROCMFP8 + sizeof(uint8_t), "wrong rocmfp8 block size/padding"); +#else +_Static_assert(sizeof(block_rocmfp2) == QS_ROCMFP2 + 2*sizeof(uint8_t), "wrong rocmfp2 block size/padding"); +_Static_assert(sizeof(block_rocmfp3) == QS_ROCMFP3 + 2*sizeof(uint8_t), "wrong rocmfp3 block size/padding"); +_Static_assert(sizeof(block_rocmfp6) == QS_ROCMFP6 + 2*sizeof(uint8_t), "wrong rocmfp6 block size/padding"); +_Static_assert(sizeof(block_rocmfp8) == QS_ROCMFP8 + sizeof(uint8_t), "wrong rocmfp8 block size/padding"); +#endif + +GGML_API float rocmfpx_ue4m3_to_fp32(uint8_t e); +GGML_API bool rocmfpx_scale_is_valid(uint8_t e); +GGML_API size_t rocmfpx_row_size_fp2(int64_t k); +GGML_API size_t rocmfpx_row_size_fp3(int64_t k); +GGML_API size_t rocmfpx_row_size_fp6(int64_t k); +GGML_API size_t rocmfpx_row_size_fp8(int64_t k); + +GGML_API void rocmfpx_quantize_row_fp2_ref(const float * GGML_RESTRICT x, block_rocmfp2 * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_dequantize_row_fp2(const block_rocmfp2 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_quantize_row_fp2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +GGML_API size_t rocmfpx_quantize_fp2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +GGML_API void rocmfpx_quantize_row_fp3_ref(const float * GGML_RESTRICT x, block_rocmfp3 * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_dequantize_row_fp3(const block_rocmfp3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_quantize_row_fp3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +GGML_API size_t rocmfpx_quantize_fp3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +GGML_API void rocmfpx_quantize_row_fp6_ref(const float * GGML_RESTRICT x, block_rocmfp6 * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_dequantize_row_fp6(const block_rocmfp6 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_quantize_row_fp6(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +GGML_API size_t rocmfpx_quantize_fp6(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +GGML_API void rocmfpx_quantize_row_fp8_ref(const float * GGML_RESTRICT x, block_rocmfp8 * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_dequantize_row_fp8(const block_rocmfp8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void rocmfpx_quantize_row_fp8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +GGML_API size_t rocmfpx_quantize_fp8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +GGML_API bool rocmfpx_validate_row_data_fp2(const void * data, size_t nbytes); +GGML_API bool rocmfpx_validate_row_data_fp3(const void * data, size_t nbytes); +GGML_API bool rocmfpx_validate_row_data_fp6(const void * data, size_t nbytes); +GGML_API bool rocmfpx_validate_row_data_fp8(const void * data, size_t nbytes); + +#ifdef __cplusplus +} +#endif diff --git a/server/deps/llama.cpp/ggml/rocmfpx/test_rocmfpx.c b/server/deps/llama.cpp/ggml/rocmfpx/test_rocmfpx.c new file mode 100644 index 000000000..5bae72bd7 --- /dev/null +++ b/server/deps/llama.cpp/ggml/rocmfpx/test_rocmfpx.c @@ -0,0 +1,164 @@ +#include "rocmfpx.h" + +#include +#include +#include + +static void fill_row(float * x, int n) { + for (int i = 0; i < n; ++i) { + const float wave = 0.75f*sinf((float) i * 0.37f) + 0.25f*cosf((float) i * 0.13f); + const float ramp = ((float) (i % 11) - 5.0f) * 0.035f; + x[i] = wave + ramp; + } + + x[7] = 3.25f; + x[19] = -2.75f; + x[43] = 1.875f; +} + +static float mse(const float * a, const float * b, int n) { + float err = 0.0f; + + for (int i = 0; i < n; ++i) { + const float d = a[i] - b[i]; + err += d*d; + } + + return err / (float) n; +} + +static float weighted_mse(const float * a, const float * b, const float * w, int n) { + float err = 0.0f; + float sum_w = 0.0f; + + for (int i = 0; i < n; ++i) { + const float d = a[i] - b[i]; + err += w[i]*d*d; + sum_w += w[i]; + } + + return sum_w > 0.0f ? err / sum_w : 0.0f; +} + +static void check_weighted_imatrix_fp3(void) { + enum { N = QK_ROCMFP3 }; + + float src[N]; + float imatrix[N]; + float plain[N]; + float weighted[N]; + block_rocmfp3 q_plain[N / QK_ROCMFP3]; + block_rocmfp3 q_weighted[N / QK_ROCMFP3]; + + for (int i = 0; i < N; ++i) { + src[i] = (i % 2) ? 0.21f : -0.21f; + imatrix[i] = 100.0f; + } + + src[0] = 9.0f; + imatrix[0] = 0.0f; + + rocmfpx_quantize_fp3(src, q_plain, 1, N, NULL); + rocmfpx_quantize_fp3(src, q_weighted, 1, N, imatrix); + rocmfpx_dequantize_row_fp3(q_plain, plain, N); + rocmfpx_dequantize_row_fp3(q_weighted, weighted, N); + + const float plain_err = weighted_mse(src, plain, imatrix, N); + const float weighted_err = weighted_mse(src, weighted, imatrix, N); + + printf("ROCmFP3 imatrix weighted_mse: plain=%g weighted=%g\n", plain_err, weighted_err); + assert(weighted_err < plain_err); +} + +static void check_weighted_imatrix_fp2(void) { + enum { N = QK_ROCMFP2 }; + + float src[N]; + float imatrix[N]; + float plain[N]; + float weighted[N]; + block_rocmfp2 q_plain[N / QK_ROCMFP2]; + block_rocmfp2 q_weighted[N / QK_ROCMFP2]; + + for (int i = 0; i < N; ++i) { + src[i] = (i % 2) ? 0.21f : -0.21f; + imatrix[i] = 100.0f; + } + + src[0] = 9.0f; + imatrix[0] = 0.0f; + + rocmfpx_quantize_fp2(src, q_plain, 1, N, NULL); + rocmfpx_quantize_fp2(src, q_weighted, 1, N, imatrix); + rocmfpx_dequantize_row_fp2(q_plain, plain, N); + rocmfpx_dequantize_row_fp2(q_weighted, weighted, N); + + const float plain_err = weighted_mse(src, plain, imatrix, N); + const float weighted_err = weighted_mse(src, weighted, imatrix, N); + + printf("ROCmFP2 imatrix weighted_mse: plain=%g weighted=%g\n", plain_err, weighted_err); + assert(weighted_err < plain_err); +} + +int main(void) { + enum { N = 64 }; + + float src[N]; + float fp2[N]; + float fp3[N]; + float fp6[N]; + float fp8[N]; + + block_rocmfp2 q2[N / QK_ROCMFP2]; + block_rocmfp3 q3[N / QK_ROCMFP3]; + block_rocmfp6 q6[N / QK_ROCMFP6]; + block_rocmfp8 q8[N / QK_ROCMFP8]; + + fill_row(src, N); + + rocmfpx_quantize_row_fp2_ref(src, q2, N); + rocmfpx_quantize_row_fp3_ref(src, q3, N); + rocmfpx_quantize_row_fp6_ref(src, q6, N); + rocmfpx_quantize_row_fp8_ref(src, q8, N); + + assert(rocmfpx_validate_row_data_fp2(q2, sizeof(q2))); + assert(rocmfpx_validate_row_data_fp3(q3, sizeof(q3))); + assert(rocmfpx_validate_row_data_fp6(q6, sizeof(q6))); + assert(rocmfpx_validate_row_data_fp8(q8, sizeof(q8))); + + rocmfpx_dequantize_row_fp2(q2, fp2, N); + rocmfpx_dequantize_row_fp3(q3, fp3, N); + rocmfpx_dequantize_row_fp6(q6, fp6, N); + rocmfpx_dequantize_row_fp8(q8, fp8, N); + + const float mse2 = mse(src, fp2, N); + const float mse3 = mse(src, fp3, N); + const float mse6 = mse(src, fp6, N); + const float mse8 = mse(src, fp8, N); + + printf("ROCmFP2: block=%zu row=%zu bpw=%.2f mse=%g\n", + sizeof(block_rocmfp2), rocmfpx_row_size_fp2(N), + 8.0f*(float) sizeof(block_rocmfp2)/(float) QK_ROCMFP2, mse2); + printf("ROCmFP3: block=%zu row=%zu bpw=%.2f mse=%g\n", + sizeof(block_rocmfp3), rocmfpx_row_size_fp3(N), + 8.0f*(float) sizeof(block_rocmfp3)/(float) QK_ROCMFP3, mse3); + printf("ROCmFP6: block=%zu row=%zu bpw=%.2f mse=%g\n", + sizeof(block_rocmfp6), rocmfpx_row_size_fp6(N), + 8.0f*(float) sizeof(block_rocmfp6)/(float) QK_ROCMFP6, mse6); + printf("ROCmFP8: block=%zu row=%zu bpw=%.2f mse=%g\n", + sizeof(block_rocmfp8), rocmfpx_row_size_fp8(N), + 8.0f*(float) sizeof(block_rocmfp8)/(float) QK_ROCMFP8, mse8); + + assert(sizeof(block_rocmfp2) == 10); + assert(isfinite(mse2)); + assert(isfinite(mse3)); + assert(isfinite(mse6)); + assert(isfinite(mse8)); + assert(mse8 < mse6); + assert(mse6 < mse3); + + check_weighted_imatrix_fp2(); + check_weighted_imatrix_fp3(); + + return 0; +} diff --git a/server/deps/llama.cpp/ggml/src/CMakeLists.txt b/server/deps/llama.cpp/ggml/src/CMakeLists.txt index 48fbe208d..315eafe2e 100644 --- a/server/deps/llama.cpp/ggml/src/CMakeLists.txt +++ b/server/deps/llama.cpp/ggml/src/CMakeLists.txt @@ -205,6 +205,10 @@ add_library(ggml-base ggml-threading.cpp ggml-threading.h ggml-quants.c + ../rocmfp4/rocmfp4.c + ../rocmfp4/rocmfp4.h + ../rocmfpx/rocmfpx.c + ../rocmfpx/rocmfpx.h ggml-quants.h gguf.cpp) diff --git a/server/deps/llama.cpp/ggml/src/ggml-common.h b/server/deps/llama.cpp/ggml/src/ggml-common.h index d47c97696..e52a2005d 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-common.h +++ b/server/deps/llama.cpp/ggml/src/ggml-common.h @@ -1132,6 +1132,12 @@ GGML_TABLE_BEGIN(int8_t, kvalues_mxfp4, 16) 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, GGML_TABLE_END() +// ROCmFP4 uses an E2M1-derived value set with the largest level retuned from +// 12 to 10, plus dual half-block UE4M3 scales. +GGML_TABLE_BEGIN(int8_t, kvalues_rocmfp4, 16) + 0, 1, 2, 3, 4, 6, 8, 10, 0, -1, -2, -3, -4, -6, -8, -10, +GGML_TABLE_END() + #define NGRID_IQ1S 2048 #define IQ1S_DELTA 0.125f #define IQ1M_DELTA 0.125f diff --git a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c index f0e14a006..170f1e4a9 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +++ b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c @@ -1826,6 +1826,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { GGML_ABORT("GGML_OP_MOE_FUSED is only implemented for CUDA"); } + case GGML_OP_DS4_HC: + { + GGML_ABORT("GGML_OP_DS4_HC is only implemented for CUDA"); + } case GGML_OP_OUT_PROD: { ggml_compute_forward_out_prod(params, tensor); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp b/server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp index c8f811406..6dff19338 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp +++ b/server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp @@ -3271,6 +3271,80 @@ static void ggml_compute_forward_swiglu( } } +// ggml_compute_forward_swiglu_ds4 + +static void ggml_compute_forward_swiglu_ds4_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + GGML_ASSERT(src1); + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src0->ne[0]; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const float clamp = ggml_get_op_params_f32(dst, 2); + + const int dr = (nr + nth - 1)/nth; + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + float * gate_p = (float *) (src0_d + i1*src0_o); + float * up_p = (float *) (src1_d + i1*src1_o); + float * dst_p = (float *) ((char *) dst->data + i1*(dst->nb[1])); + + for (int k = 0; k < nc; k++) { + const float gate = std::min(gate_p[k], clamp); + const float up = std::clamp(up_p[k], -clamp, clamp); + dst_p[k] = up * gate / (1.f + expf(-gate)); + } + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = dst_p[k]; + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif // NDEBUG + } +} + +static void ggml_compute_forward_swiglu_ds4( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_swiglu_ds4_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_swiglu_oai static void ggml_compute_forward_swiglu_oai_f32( @@ -9801,6 +9875,10 @@ void ggml_compute_forward_glu( { ggml_compute_forward_swiglu_oai(params, dst); } break; + case GGML_GLU_OP_SWIGLU_DS4: + { + ggml_compute_forward_swiglu_ds4(params, dst); + } break; case GGML_GLU_OP_GEGLU_ERF: { ggml_compute_forward_geglu_erf(params, dst); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh index 9df1461a5..eb7d125a1 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh @@ -19,6 +19,8 @@ #endif #endif #include "ggml-common.h" +#include "../../rocmfp4/rocmfp4.h" +#include "../../rocmfpx/rocmfpx.h" #include #include @@ -29,6 +31,26 @@ #include #include +#ifndef GGML_ROCMFP6_EXPANDED_DEVICE +#define GGML_ROCMFP6_EXPANDED_DEVICE 0 +#endif + +// Optional device-only ROCmFP6 layout. GGUF/CPU storage remains the packed +// block_rocmfp6 layout; experimental ROCm builds may expand qs to signed +// bytes to avoid bit unpacking in hot matmul/FA kernels. +struct block_rocmfp6_expanded { + int8_t qs[QK_ROCMFP6]; + uint8_t e[2]; +}; + +static_assert(sizeof(block_rocmfp6_expanded) == QK_ROCMFP6 + 2*sizeof(uint8_t), "wrong expanded rocmfp6 block size/padding"); + +#if GGML_ROCMFP6_EXPANDED_DEVICE +using block_rocmfp6_device = block_rocmfp6_expanded; +#else +using block_rocmfp6_device = block_rocmfp6; +#endif + #if defined(GGML_USE_HIP) #include "vendors/hip.h" #elif defined(GGML_USE_MUSA) @@ -959,6 +981,48 @@ struct ggml_cuda_type_traits { static constexpr int qi = QI8_0; }; +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_ROCMFP4; + static constexpr int qr = QR_ROCMFP4; + static constexpr int qi = QI_ROCMFP4; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_ROCMFP4; + static constexpr int qr = QR_ROCMFP4; + static constexpr int qi = QI_ROCMFP4; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_ROCMFP2; + static constexpr int qr = QR_ROCMFP2; + static constexpr int qi = QI_ROCMFP2; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_ROCMFP3; + static constexpr int qr = QR_ROCMFP3; + static constexpr int qi = QI_ROCMFP3; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_ROCMFP6; + static constexpr int qr = QR_ROCMFP6; + static constexpr int qi = QI_ROCMFP6; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_ROCMFP8; + static constexpr int qr = QR_ROCMFP8; + static constexpr int qi = QI_ROCMFP8; +}; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK_MXFP4; @@ -1466,10 +1530,14 @@ struct ggml_cuda_mm_fusion_args_host { const ggml_tensor * gate = nullptr; const ggml_tensor * gate_bias = nullptr; ggml_glu_op glu_op; + float glu_param0 = 0.0f; + float glu_param1 = 0.0f; }; struct ggml_cuda_mm_fusion_args_device { const void * x_bias = nullptr; const void * gate = nullptr; const void * gate_bias = nullptr; ggml_glu_op glu_op; + float glu_param0 = 0.0f; + float glu_param1 = 0.0f; }; diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/concat.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/concat.cu index e9ffd274b..51a3efa53 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/concat.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/concat.cu @@ -1,7 +1,8 @@ #include "concat.cuh" // contiguous kernels -static __global__ void concat_f32_dim0(const float * x, const float * y, float * dst, const int ne0, const int ne00) { +template +static __global__ void concat_dim0(const T * x, const T * y, T * dst, const int ne0, const int ne00) { int nidx = threadIdx.x + blockIdx.x * blockDim.x; if (nidx >= ne0) { return; @@ -27,7 +28,8 @@ static __global__ void concat_f32_dim0(const float * x, const float * y, float * } } -static __global__ void concat_f32_dim1(const float * x, const float * y, float * dst, const int ne0, const int ne01) { +template +static __global__ void concat_dim1(const T * x, const T * y, T * dst, const int ne0, const int ne01) { int nidx = threadIdx.x + blockIdx.x * blockDim.x; if (nidx >= ne0) { return; @@ -53,7 +55,8 @@ static __global__ void concat_f32_dim1(const float * x, const float * y, float * } } -static __global__ void concat_f32_dim2(const float * x, const float * y, float * dst, const int ne0, const int ne02) { +template +static __global__ void concat_dim2(const T * x, const T * y, T * dst, const int ne0, const int ne02) { int nidx = threadIdx.x + blockIdx.x * blockDim.x; if (nidx >= ne0) { return; @@ -79,24 +82,25 @@ static __global__ void concat_f32_dim2(const float * x, const float * y, float * } } -static void concat_f32_cuda(const float * x, const float * y, float * dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, cudaStream_t stream) { +template +static void concat_cuda(const T * x, const T * y, T * dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, cudaStream_t stream) { int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE; dim3 gridDim(num_blocks, ne1, ne2); if (dim == 0) { - concat_f32_dim0<<>>(x, y, dst, ne0, ne00); + concat_dim0<<>>(x, y, dst, ne0, ne00); return; } if (dim == 1) { - concat_f32_dim1<<>>(x, y, dst, ne0, ne01); + concat_dim1<<>>(x, y, dst, ne0, ne01); return; } - concat_f32_dim2<<>>(x, y, dst, ne0, ne02); + concat_dim2<<>>(x, y, dst, ne0, ne02); } // non-contiguous kernel (slow) -template +template static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) - concat_f32_non_cont( + concat_non_cont( const char * src0, const char * src1, char * dst, @@ -130,54 +134,44 @@ static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) const int64_t i2 = blockIdx.y; const int64_t i1 = blockIdx.x; - const float * x; + const T * x; for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) { if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); + x = (const T *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); } else { if constexpr (dim == 0) { - x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + i1 * nb11 + (i0 - ne00) * nb10); + x = (const T *) (src1 + i3 * nb13 + i2 * nb12 + i1 * nb11 + (i0 - ne00) * nb10); } else if constexpr (dim == 1) { - x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + (i1 - ne01) * nb11 + i0 * nb10); + x = (const T *) (src1 + i3 * nb13 + i2 * nb12 + (i1 - ne01) * nb11 + i0 * nb10); } else if constexpr (dim == 2) { - x = (const float *) (src1 + i3 * nb13 + (i2 - ne02) * nb12 + i1 * nb11 + i0 * nb10); + x = (const T *) (src1 + i3 * nb13 + (i2 - ne02) * nb12 + i1 * nb11 + i0 * nb10); } else if constexpr (dim == 3) { - x = (const float *) (src1 + (i3 - ne03) * nb13 + i2 * nb12 + i1 * nb11 + i0 * nb10); + x = (const T *) (src1 + (i3 - ne03) * nb13 + i2 * nb12 + i1 * nb11 + i0 * nb10); } } - float * y = (float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + T * y = (T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); *y = *x; } } - -void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - +template +static void concat_cuda_typed(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, int dim) { cudaStream_t stream = ctx.stream(); - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const float * src0_d = (const float *)src0->data; - const float * src1_d = (const float *)src1->data; - - float * dst_d = (float *)dst->data; + const T * src0_d = (const T *)src0->data; + const T * src1_d = (const T *)src1->data; + T * dst_d = (T *)dst->data; if (dim != 3) { for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_f32_cuda( - src0_d + i3 * (src0->nb[3] / 4), - src1_d + i3 * (src1->nb[3] / 4), - dst_d + i3 * ( dst->nb[3] / 4), + concat_cuda( + src0_d + i3 * (src0->nb[3] / sizeof(T)), + src1_d + i3 * (src1->nb[3] / sizeof(T)), + dst_d + i3 * ( dst->nb[3] / sizeof(T)), src0->ne[0], src0->ne[1], src0->ne[2], dst->ne[0], dst->ne[1], dst->ne[2], dim, stream); } @@ -185,13 +179,13 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const size_t size0 = ggml_nbytes(src0); const size_t size1 = ggml_nbytes(src1); - CUDA_CHECK(cudaMemcpyAsync(dst_d, src0_d, size0, cudaMemcpyDeviceToDevice, stream)); - CUDA_CHECK(cudaMemcpyAsync(dst_d + size0/4, src1_d, size1, cudaMemcpyDeviceToDevice, stream)); + CUDA_CHECK(cudaMemcpyAsync(dst_d, src0_d, size0, cudaMemcpyDeviceToDevice, stream)); + CUDA_CHECK(cudaMemcpyAsync((char *)dst_d + size0, src1_d, size1, cudaMemcpyDeviceToDevice, stream)); } } else { dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]); auto launch_kernel = [&](auto dim) { - concat_f32_non_cont<<>>( + concat_non_cont<<>>( (const char *) src0->data, (const char *) src1->data, (char *) dst->data, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], @@ -219,3 +213,30 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { } } } + +void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + const int32_t dim = ((int32_t *) dst->op_params)[0]; + + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + + switch (src0->type) { + case GGML_TYPE_F32: + concat_cuda_typed(ctx, src0, src1, dst, dim); + break; + case GGML_TYPE_F16: + concat_cuda_typed(ctx, src0, src1, dst, dim); + break; + case GGML_TYPE_BF16: + concat_cuda_typed(ctx, src0, src1, dst, dim); + break; + case GGML_TYPE_I8: + concat_cuda_typed(ctx, src0, src1, dst, dim); + break; + default: + GGML_ABORT("unsupported concat type %s", ggml_type_name(src0->type)); + } +} diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/convert.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/convert.cu index c675034af..28d042aa0 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/convert.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/convert.cu @@ -1,6 +1,7 @@ #include "convert.cuh" #include "dequantize.cuh" #include "tq3-quant.cuh" +#include "../../rocmfp4/rocmfp4_hip_scale.cuh" #include @@ -487,6 +488,43 @@ static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_ } } +template +static __global__ void dequantize_block_rocmfp4(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int64_t i = blockIdx.x; + const block_rocmfp4 * x = (const block_rocmfp4 *) vx + i*(QK_K/QK_ROCMFP4); + + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 4*il; + const uint8_t * q4 = x[ib].qs + 4*il; + const float d0 = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e[0]); + const float d1 = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e[1]); + for (int j = 0; j < 4; ++j) { + y[j+ 0] = d0 * rocmfp4_decode_i8(q4[j]); + y[j+16] = d1 * rocmfp4_decode_i8(q4[j] >> 4); + } +} + +template +static __global__ void dequantize_block_rocmfp4_fast(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int64_t i = blockIdx.x; + const block_rocmfp4_fast * x = (const block_rocmfp4_fast *) vx + i*(QK_K/QK_ROCMFP4); + + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 4*il; + const uint8_t * q4 = x[ib].qs + 4*il; + const float d = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e); + for (int j = 0; j < 4; ++j) { + y[j+ 0] = d * rocmfp4_decode_i8(q4[j]); + y[j+16] = d * rocmfp4_decode_i8(q4[j] >> 4); + } +} + template static void dequantize_block_cuda(const void * vx, dst_t * y, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03, @@ -645,6 +683,18 @@ static void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t dequantize_block_mxfp4<<>>(vx, y); } +template +static void dequantize_row_rocmfp4_hip(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_rocmfp4<<>>(vx, y); +} + +template +static void dequantize_row_rocmfp4_fast_hip(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_rocmfp4_fast<<>>(vx, y); +} + template static __global__ void dequantize_block_nvfp4( const void * __restrict__ vx, @@ -786,6 +836,18 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_Q4_0_ROCMFP4: + return dequantize_row_rocmfp4_hip; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + return dequantize_row_rocmfp4_fast_hip; + case GGML_TYPE_Q2_0_ROCMFP2: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q3_0_ROCMFPX: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q6_0_ROCMFPX: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q8_0_ROCMFPX: + return dequantize_block_cont_cuda; case GGML_TYPE_F32: return convert_unary_cont_cuda; case GGML_TYPE_BF16: @@ -841,6 +903,18 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_Q4_0_ROCMFP4: + return dequantize_row_rocmfp4_hip; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + return dequantize_row_rocmfp4_fast_hip; + case GGML_TYPE_Q2_0_ROCMFP2: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q3_0_ROCMFPX: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q6_0_ROCMFPX: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q8_0_ROCMFPX: + return dequantize_block_cont_cuda; case GGML_TYPE_F16: return convert_unary_cont_cuda; case GGML_TYPE_BF16: @@ -864,6 +938,18 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: return dequantize_block_cuda; + case GGML_TYPE_Q4_0_ROCMFP4: + return dequantize_block_cuda; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + return dequantize_block_cuda; + case GGML_TYPE_Q2_0_ROCMFP2: + return dequantize_block_cuda; + case GGML_TYPE_Q3_0_ROCMFPX: + return dequantize_block_cuda; + case GGML_TYPE_Q6_0_ROCMFPX: + return dequantize_block_cuda; + case GGML_TYPE_Q8_0_ROCMFPX: + return dequantize_block_cuda; case GGML_TYPE_BF16: return convert_unary_cuda; default: @@ -885,6 +971,18 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: return dequantize_block_cuda; + case GGML_TYPE_Q4_0_ROCMFP4: + return dequantize_block_cuda; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + return dequantize_block_cuda; + case GGML_TYPE_Q2_0_ROCMFP2: + return dequantize_block_cuda; + case GGML_TYPE_Q3_0_ROCMFPX: + return dequantize_block_cuda; + case GGML_TYPE_Q6_0_ROCMFPX: + return dequantize_block_cuda; + case GGML_TYPE_Q8_0_ROCMFPX: + return dequantize_block_cuda; case GGML_TYPE_F16: return convert_unary_cuda; default: @@ -906,6 +1004,18 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: return dequantize_block_cuda; + case GGML_TYPE_Q4_0_ROCMFP4: + return dequantize_block_cuda; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + return dequantize_block_cuda; + case GGML_TYPE_Q2_0_ROCMFP2: + return dequantize_block_cuda; + case GGML_TYPE_Q3_0_ROCMFPX: + return dequantize_block_cuda; + case GGML_TYPE_Q6_0_ROCMFPX: + return dequantize_block_cuda; + case GGML_TYPE_Q8_0_ROCMFPX: + return dequantize_block_cuda; case GGML_TYPE_BF16: return convert_unary_cuda; default: diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu index 372fc98b1..3d7a698b9 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu @@ -1,6 +1,7 @@ #include "cpy.cuh" #include "dequantize.cuh" #include "cpy-utils.cuh" +#include "../../rocmfp4/rocmfp4_hip_scale.cuh" #if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY) #include "ggml-musa/mudnn.cuh" #endif // GGML_USE_MUSA && GGML_MUSA_MUDNN_COPY @@ -119,6 +120,72 @@ static __device__ void cpy_blck_q_f32(const char * cxi, char * cdsti) { } } +static __device__ void cpy_blck_rocmfp4_f32(const char * cxi, char * cdsti) { + const block_rocmfp4 * x = (const block_rocmfp4 *) cxi; + float * cdstf = (float *) cdsti; + + const float d0 = rocmfp4_ue4m3_to_fp32_half_finite(x->e[0]); + const float d1 = rocmfp4_ue4m3_to_fp32_half_finite(x->e[1]); + +#pragma unroll + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q = x->qs[j]; + cdstf[j] = d0 * (float) rocmfp4_decode_i8(q); + cdstf[j + QK_ROCMFP4/2] = d1 * (float) rocmfp4_decode_i8(q >> 4); + } +} + +static __device__ void cpy_blck_rocmfp4_fast_f32(const char * cxi, char * cdsti) { + const block_rocmfp4_fast * x = (const block_rocmfp4_fast *) cxi; + float * cdstf = (float *) cdsti; + + const float d = rocmfp4_ue4m3_to_fp32_half_finite(x->e); + +#pragma unroll + for (int j = 0; j < QK_ROCMFP4/2; ++j) { + const uint8_t q = x->qs[j]; + cdstf[j] = d * (float) rocmfp4_decode_i8(q); + cdstf[j + QK_ROCMFP4/2] = d * (float) rocmfp4_decode_i8(q >> 4); + } +} + +static __global__ void cpy_rocmfp4_f32_contiguous(const block_rocmfp4 * cx, float * cdst, const int64_t ne) { + const int64_t packed_idx = (int64_t)blockDim.x * blockIdx.x + threadIdx.x; + const int64_t packed_count = (ne / QK_ROCMFP4) * (QK_ROCMFP4/2); + + if (packed_idx >= packed_count) { + return; + } + + const int64_t ib = packed_idx >> 4; + const int j = packed_idx & 0x0f; + const int64_t base = ib * QK_ROCMFP4; + const uint8_t q = cx[ib].qs[j]; + const float d0 = rocmfp4_ue4m3_to_fp32_half_finite(cx[ib].e[0]); + const float d1 = rocmfp4_ue4m3_to_fp32_half_finite(cx[ib].e[1]); + + cdst[base + j] = d0 * (float) rocmfp4_decode_i8(q); + cdst[base + j + QK_ROCMFP4/2] = d1 * (float) rocmfp4_decode_i8(q >> 4); +} + +static __global__ void cpy_rocmfp4_fast_f32_contiguous(const block_rocmfp4_fast * cx, float * cdst, const int64_t ne) { + const int64_t packed_idx = (int64_t)blockDim.x * blockIdx.x + threadIdx.x; + const int64_t packed_count = (ne / QK_ROCMFP4) * (QK_ROCMFP4/2); + + if (packed_idx >= packed_count) { + return; + } + + const int64_t ib = packed_idx >> 4; + const int j = packed_idx & 0x0f; + const int64_t base = ib * QK_ROCMFP4; + const uint8_t q = cx[ib].qs[j]; + const float d = rocmfp4_ue4m3_to_fp32_half_finite(cx[ib].e); + + cdst[base + j] = d * (float) rocmfp4_decode_i8(q); + cdst[base + j + QK_ROCMFP4/2] = d * (float) rocmfp4_decode_i8(q >> 4); +} + template static __global__ void cpy_f32_q(const char * cx, char * cdst, const int64_t ne, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, @@ -308,6 +375,56 @@ static void ggml_cpy_q4_1_f32_cuda( ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +static void ggml_cpy_rocmfp4_f32_hip( + const char * cx, char * cdst, const int64_t ne, + const int64_t ne00, const int64_t ne01, const int64_t ne02, + const int64_t nb00, const int64_t nb01, const int64_t nb02, + const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, + const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, + cudaStream_t stream) { + GGML_ASSERT(ne % QK_ROCMFP4 == 0); + const int64_t num_blocks = ne / QK_ROCMFP4; + GGML_ASSERT(num_blocks < UINT_MAX); + cpy_q_f32<<>>( + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + +static void ggml_cpy_rocmfp4_f32_contiguous_hip( + const char * cx, char * cdst, const int64_t ne, cudaStream_t stream) { + GGML_ASSERT(ne % QK_ROCMFP4 == 0); + const int64_t packed_count = (ne / QK_ROCMFP4) * (QK_ROCMFP4/2); + const int64_t num_blocks = (packed_count + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks < UINT_MAX); + cpy_rocmfp4_f32_contiguous<<>>( + (const block_rocmfp4 *) cx, (float *) cdst, ne); +} + +static void ggml_cpy_rocmfp4_fast_f32_hip( + const char * cx, char * cdst, const int64_t ne, + const int64_t ne00, const int64_t ne01, const int64_t ne02, + const int64_t nb00, const int64_t nb01, const int64_t nb02, + const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, + const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, + cudaStream_t stream) { + GGML_ASSERT(ne % QK_ROCMFP4 == 0); + const int64_t num_blocks = ne / QK_ROCMFP4; + GGML_ASSERT(num_blocks < UINT_MAX); + cpy_q_f32<<>>( + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + +static void ggml_cpy_rocmfp4_fast_f32_contiguous_hip( + const char * cx, char * cdst, const int64_t ne, cudaStream_t stream) { + GGML_ASSERT(ne % QK_ROCMFP4 == 0); + const int64_t packed_count = (ne / QK_ROCMFP4) * (QK_ROCMFP4/2); + const int64_t num_blocks = (packed_count + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks < UINT_MAX); + cpy_rocmfp4_fast_f32_contiguous<<>>( + (const block_rocmfp4_fast *) cx, (float *) cdst, ne); +} + static void ggml_cpy_f32_q5_0_cuda( const char * cx, char * cdst, const int64_t ne, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, @@ -556,6 +673,20 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) { ggml_cpy_q4_0_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q4_0_ROCMFP4 && src1->type == GGML_TYPE_F32) { + if (contiguous_srcs) { + ggml_cpy_rocmfp4_f32_contiguous_hip(src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_rocmfp4_f32_hip + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_Q4_0_ROCMFP4_FAST && src1->type == GGML_TYPE_F32) { + if (contiguous_srcs) { + ggml_cpy_rocmfp4_fast_f32_contiguous_hip(src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_rocmfp4_fast_f32_hip + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) { ggml_cpy_f32_q4_1_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh index 1801bd048..dc660e1aa 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh @@ -1,5 +1,6 @@ #include "common.cuh" #include "tq3-quant.cuh" +#include "../../rocmfp4/rocmfp4_hip_scale.cuh" static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ const block_q4_0 * x = (const block_q4_0 *) vx; @@ -76,3 +77,125 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in v.x *= d; v.y *= d; } + +static __device__ __forceinline__ void dequantize_rocmfp4(const void * vx, const int64_t ib, const int iqs, float2 & v) { + const block_rocmfp4 * x = (const block_rocmfp4 *) vx; + + const int q = x[ib].qs[iqs]; + const float d0 = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e[0]); + const float d1 = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e[1]); + + v.x = d0 * rocmfp4_decode_i8(q); + v.y = d1 * rocmfp4_decode_i8(q >> 4); +} + +static __device__ __forceinline__ void dequantize_rocmfp4_fast(const void * vx, const int64_t ib, const int iqs, float2 & v) { + const block_rocmfp4_fast * x = (const block_rocmfp4_fast *) vx; + + const int q = x[ib].qs[iqs]; + const float d = rocmfp4_ue4m3_to_fp32_half_finite(x[ib].e); + + v.x = d * rocmfp4_decode_i8(q); + v.y = d * rocmfp4_decode_i8(q >> 4); +} + +template +static __device__ __forceinline__ uint32_t rocmfpx_load_qs_window_cuda(const uint8_t * src, const int byte_pos) { + uint32_t v = (uint32_t) src[byte_pos + 0]; + + if (byte_pos + 1 < qs) { + v |= (uint32_t) src[byte_pos + 1] << 8; + } + if (byte_pos + 2 < qs) { + v |= (uint32_t) src[byte_pos + 2] << 16; + } + + return v; +} + +static __device__ __forceinline__ uint32_t rocmfpx_get_fp3_code_cuda(const uint8_t * src, const int i) { + const int bit_pos = i * 3; + const int byte_pos = bit_pos >> 3; + const int shift = bit_pos & 7; + return (rocmfpx_load_qs_window_cuda(src, byte_pos) >> shift) & 7u; +} + +static __device__ __forceinline__ uint32_t rocmfpx_get_fp2_code_cuda(const uint8_t * src, const int i) { + return (src[i >> 2] >> (2*(i & 3))) & 3u; +} + +static __device__ __forceinline__ uint32_t rocmfpx_get_fp6_code_cuda(const uint8_t * src, const int i) { + const int bit_pos = i * 6; + const int byte_pos = bit_pos >> 3; + const int shift = bit_pos & 7; + return (rocmfpx_load_qs_window_cuda(src, byte_pos) >> shift) & 63u; +} + +static __device__ __forceinline__ int rocmfpx_decode_fp3_code_cuda(const uint32_t code) { + const uint32_t mag_code = code & 3u; + const int mag = mag_code == 3u ? 4 : (int) mag_code; + return (code & 4u) ? -mag : mag; +} + +static __device__ __forceinline__ float rocmfpx_decode_fp2_code_cuda(const uint32_t code) { + switch (code & 3u) { + case 0: return (float) ROCMFP2_KVALUE_0_I8; + case 1: return (float) ROCMFP2_KVALUE_1_I8; + case 2: return (float) ROCMFP2_KVALUE_2_I8; + default: return (float) ROCMFP2_KVALUE_3_I8; + } +} + +static __device__ __forceinline__ int rocmfpx_decode_fp6_code_cuda(const uint32_t code) { + const int mag = (int) (code & 31u); + return (code & 32u) ? -mag : mag; +} + +static __device__ __forceinline__ void dequantize_rocmfpx_fp3(const void * vx, const int64_t ib, const int iqs, float2 & v) { + const block_rocmfp3 * x = (const block_rocmfp3 *) vx; + + const int i0 = iqs + 0; + const int i1 = iqs + 1; + const float d0 = rocmfpx_ue4m3_to_fp32_finite(x[ib].e[i0 >= QK_ROCMFP3/2]); + const float d1 = rocmfpx_ue4m3_to_fp32_finite(x[ib].e[i1 >= QK_ROCMFP3/2]); + + v.x = d0 * (float) rocmfpx_decode_fp3_code_cuda(rocmfpx_get_fp3_code_cuda(x[ib].qs, i0)); + v.y = d1 * (float) rocmfpx_decode_fp3_code_cuda(rocmfpx_get_fp3_code_cuda(x[ib].qs, i1)); +} + +static __device__ __forceinline__ void dequantize_rocmfpx_fp2(const void * vx, const int64_t ib, const int iqs, float2 & v) { + const block_rocmfp2 * x = (const block_rocmfp2 *) vx; + + const int i0 = iqs + 0; + const int i1 = iqs + 1; + const float d0 = rocmfpx_ue4m3_to_fp32_finite(x[ib].e[i0 >= QK_ROCMFP2/2]); + const float d1 = rocmfpx_ue4m3_to_fp32_finite(x[ib].e[i1 >= QK_ROCMFP2/2]); + + v.x = d0 * rocmfpx_decode_fp2_code_cuda(rocmfpx_get_fp2_code_cuda(x[ib].qs, i0)); + v.y = d1 * rocmfpx_decode_fp2_code_cuda(rocmfpx_get_fp2_code_cuda(x[ib].qs, i1)); +} + +static __device__ __forceinline__ void dequantize_rocmfpx_fp6(const void * vx, const int64_t ib, const int iqs, float2 & v) { + const block_rocmfp6_device * x = (const block_rocmfp6_device *) vx; + + const int i0 = iqs + 0; + const int i1 = iqs + 1; + const float d0 = rocmfpx_ue4m3_to_fp32_finite(x[ib].e[i0 >= QK_ROCMFP6/2]); + const float d1 = rocmfpx_ue4m3_to_fp32_finite(x[ib].e[i1 >= QK_ROCMFP6/2]); + +#if GGML_ROCMFP6_EXPANDED_DEVICE + v.x = d0 * (float) x[ib].qs[i0]; + v.y = d1 * (float) x[ib].qs[i1]; +#else + v.x = d0 * (float) rocmfpx_decode_fp6_code_cuda(rocmfpx_get_fp6_code_cuda(x[ib].qs, i0)); + v.y = d1 * (float) rocmfpx_decode_fp6_code_cuda(rocmfpx_get_fp6_code_cuda(x[ib].qs, i1)); +#endif +} + +static __device__ __forceinline__ void dequantize_rocmfpx_fp8(const void * vx, const int64_t ib, const int iqs, float2 & v) { + const block_rocmfp8 * x = (const block_rocmfp8 *) vx; + + const float d = rocmfpx_ue4m3_to_fp32_finite(x[ib].e); + v.x = d * (float) x[ib].qs[iqs + 0]; + v.y = d * (float) x[ib].qs[iqs + 1]; +} diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cu new file mode 100644 index 000000000..1c6b86972 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cu @@ -0,0 +1,367 @@ +#include "ds4-hc.cuh" + +// Fused DeepSeek4 hyper-connection ops. +// +// mode 0 (pre): src0 = mix [mix_dim] (f32, from fn @ rms_norm(hc_state)) +// src1 = base [mix_dim] (f32) +// src2 = hc_state [n_embd*n_hc] (f32, raw residual streams) +// dst = [n_embd + mix_dim]: +// dst[0..n_embd) = working vector (pre-mixed input) +// dst[n_embd..n_embd+mix) = split = {pre[n_hc], post[n_hc], comb[n_hc*n_hc]} +// Math matches cpu_hc_sinkhorn + finish_hc_pre_from_mix_into in +// deepseek4_graph.cpp (sigmoid gates + Sinkhorn-normalized combine). +// +// mode 1 (post): src0 = residual hc_state [n_embd*n_hc] +// src1 = block_out [n_embd] +// src2 = split [mix_dim] (view of a mode-0 dst tail) +// dst = new hc_state [n_embd*n_hc]: +// dst[h*n_embd+d] = post[h]*block_out[d] +// + sum_src comb[h + src*n_hc] * residual[src*n_embd+d] +// +// mode 2 (out): src0 = mix [n_hc] +// src1 = base [n_hc] +// src2 = hc_state [n_embd*n_hc] +// dst = [n_embd]: weights[h] = sigmoid(mix[h]*s0+base[h]) + 1e-6; +// dst[d] = sum_h weights[h]*hc_state[h*n_embd+d] + +#define DS4_HC_SINKHORN_EPS 1.0e-6f +#define DS4_HC_MAX_HC 8 +#define DS4_HC_MAX_MIX (2*DS4_HC_MAX_HC + DS4_HC_MAX_HC*DS4_HC_MAX_HC) + +static __device__ __forceinline__ float ds4_hc_sigmoid(float x) { + return 1.0f / (1.0f + expf(-x)); +} + +static __device__ void ds4_hc_sinkhorn_split( + const float * mix, + const float * base, + float pre_scale, + float post_scale, + float comb_scale, + int n_hc, + int iters, + float * split) { + for (int i = 0; i < n_hc; ++i) { + split[i] = ds4_hc_sigmoid(mix[i] * pre_scale + base[i]) + DS4_HC_SINKHORN_EPS; + } + for (int i = 0; i < n_hc; ++i) { + split[n_hc + i] = 2.0f * ds4_hc_sigmoid(mix[n_hc + i] * post_scale + base[n_hc + i]); + } + + float c[DS4_HC_MAX_HC * DS4_HC_MAX_HC]; + for (int dst_i = 0; dst_i < n_hc; ++dst_i) { + float row_max = -1.0e30f; + for (int src_i = 0; src_i < n_hc; ++src_i) { + const int idx = src_i + dst_i * n_hc; + const float v = mix[2 * n_hc + idx] * comb_scale + base[2 * n_hc + idx]; + c[idx] = v; + row_max = v > row_max ? v : row_max; + } + float row_sum = 0.0f; + for (int src_i = 0; src_i < n_hc; ++src_i) { + const int idx = src_i + dst_i * n_hc; + c[idx] = expf(c[idx] - row_max); + row_sum += c[idx]; + } + const float inv = 1.0f / row_sum; + for (int src_i = 0; src_i < n_hc; ++src_i) { + c[src_i + dst_i * n_hc] = c[src_i + dst_i * n_hc] * inv + DS4_HC_SINKHORN_EPS; + } + } + for (int src_i = 0; src_i < n_hc; ++src_i) { + float sum = 0.0f; + for (int dst_i = 0; dst_i < n_hc; ++dst_i) sum += c[src_i + dst_i * n_hc]; + const float inv = 1.0f / (sum + DS4_HC_SINKHORN_EPS); + for (int dst_i = 0; dst_i < n_hc; ++dst_i) c[src_i + dst_i * n_hc] *= inv; + } + for (int iter = 1; iter < iters; ++iter) { + for (int dst_i = 0; dst_i < n_hc; ++dst_i) { + float sum = 0.0f; + for (int src_i = 0; src_i < n_hc; ++src_i) sum += c[src_i + dst_i * n_hc]; + const float inv = 1.0f / (sum + DS4_HC_SINKHORN_EPS); + for (int src_i = 0; src_i < n_hc; ++src_i) c[src_i + dst_i * n_hc] *= inv; + } + for (int src_i = 0; src_i < n_hc; ++src_i) { + float sum = 0.0f; + for (int dst_i = 0; dst_i < n_hc; ++dst_i) sum += c[src_i + dst_i * n_hc]; + const float inv = 1.0f / (sum + DS4_HC_SINKHORN_EPS); + for (int dst_i = 0; dst_i < n_hc; ++dst_i) c[src_i + dst_i * n_hc] *= inv; + } + } + for (int i = 0; i < n_hc * n_hc; ++i) { + split[2 * n_hc + i] = c[i]; + } +} + +// Fully-unrolled variant: with compile-time NHC the c[] matrix lives in +// registers. The generic version's runtime-bound loops force c[] into scratch +// (private, VRAM-backed) memory, making the serial sinkhorn ~20x slower +// (97us vs 5us measured on gfx1151). +template +static __device__ void ds4_hc_sinkhorn_split_t( + const float * mix, + const float * base, + float pre_scale, + float post_scale, + float comb_scale, + int iters, + float * split) { + #pragma unroll + for (int i = 0; i < NHC; ++i) { + split[i] = ds4_hc_sigmoid(mix[i] * pre_scale + base[i]) + DS4_HC_SINKHORN_EPS; + } + #pragma unroll + for (int i = 0; i < NHC; ++i) { + split[NHC + i] = 2.0f * ds4_hc_sigmoid(mix[NHC + i] * post_scale + base[NHC + i]); + } + float c[NHC * NHC]; + #pragma unroll + for (int dst_i = 0; dst_i < NHC; ++dst_i) { + float row_max = -1.0e30f; + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) { + const int idx = src_i + dst_i * NHC; + const float v = mix[2 * NHC + idx] * comb_scale + base[2 * NHC + idx]; + c[idx] = v; + row_max = v > row_max ? v : row_max; + } + float row_sum = 0.0f; + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) { + const int idx = src_i + dst_i * NHC; + c[idx] = expf(c[idx] - row_max); + row_sum += c[idx]; + } + const float inv = 1.0f / row_sum; + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) { + c[src_i + dst_i * NHC] = c[src_i + dst_i * NHC] * inv + DS4_HC_SINKHORN_EPS; + } + } + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) { + float sum = 0.0f; + #pragma unroll + for (int dst_i = 0; dst_i < NHC; ++dst_i) sum += c[src_i + dst_i * NHC]; + const float inv = 1.0f / (sum + DS4_HC_SINKHORN_EPS); + #pragma unroll + for (int dst_i = 0; dst_i < NHC; ++dst_i) c[src_i + dst_i * NHC] *= inv; + } + for (int iter = 1; iter < iters; ++iter) { + #pragma unroll + for (int dst_i = 0; dst_i < NHC; ++dst_i) { + float sum = 0.0f; + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) sum += c[src_i + dst_i * NHC]; + const float inv = 1.0f / (sum + DS4_HC_SINKHORN_EPS); + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) c[src_i + dst_i * NHC] *= inv; + } + #pragma unroll + for (int src_i = 0; src_i < NHC; ++src_i) { + float sum = 0.0f; + #pragma unroll + for (int dst_i = 0; dst_i < NHC; ++dst_i) sum += c[src_i + dst_i * NHC]; + const float inv = 1.0f / (sum + DS4_HC_SINKHORN_EPS); + #pragma unroll + for (int dst_i = 0; dst_i < NHC; ++dst_i) c[src_i + dst_i * NHC] *= inv; + } + } + #pragma unroll + for (int i = 0; i < NHC * NHC; ++i) { + split[2 * NHC + i] = c[i]; + } +} + +template +static __global__ void ds4_hc_pre_kernel_t( + const float * __restrict__ mix, + const float * __restrict__ base, + const float * __restrict__ hc_state, + float * __restrict__ dst, + int n_embd, + int iters, + float pre_scale, + float post_scale, + float comb_scale) { + __shared__ float split[DS4_HC_MAX_MIX]; + __shared__ float s_mix[DS4_HC_MAX_MIX]; + __shared__ float s_base[DS4_HC_MAX_MIX]; + constexpr int mix_dim = 2 * NHC + NHC * NHC; + const int tid = threadIdx.x; + + if (tid < mix_dim) { + s_mix[tid] = mix[tid]; + s_base[tid] = base[tid]; + } + __syncthreads(); + + if (tid == 0) { + ds4_hc_sinkhorn_split_t(s_mix, s_base, pre_scale, post_scale, comb_scale, iters, split); + if (blockIdx.x == 0) { + #pragma unroll + for (int i = 0; i < mix_dim; ++i) { + dst[n_embd + i] = split[i]; + } + } + } + __syncthreads(); + + const int d = (int) blockIdx.x * blockDim.x + tid; + if (d < n_embd) { + float acc = 0.0f; + #pragma unroll + for (int h = 0; h < NHC; ++h) { + acc += split[h] * hc_state[(size_t) h * n_embd + d]; + } + dst[d] = acc; + } +} + +static __global__ void ds4_hc_pre_kernel( + const float * __restrict__ mix, + const float * __restrict__ base, + const float * __restrict__ hc_state, + float * __restrict__ dst, + int n_embd, + int n_hc, + int iters, + float pre_scale, + float post_scale, + float comb_scale) { + __shared__ float split[DS4_HC_MAX_MIX]; + __shared__ float s_mix[DS4_HC_MAX_MIX]; + __shared__ float s_base[DS4_HC_MAX_MIX]; + const int mix_dim = 2 * n_hc + n_hc * n_hc; + const int tid = threadIdx.x; + + // Stage mix/base cooperatively: base lives in managed (UMA) memory where + // serial scalar loads cost ~2us each; one parallel coalesced load instead. + if (tid < mix_dim) { + s_mix[tid] = mix[tid]; + s_base[tid] = base[tid]; + } + __syncthreads(); + + // Each block redoes the (tiny) sinkhorn into shared memory so the mix + // loop below can spread across the whole GPU instead of one CU. + if (tid == 0) { + ds4_hc_sinkhorn_split(s_mix, s_base, pre_scale, post_scale, comb_scale, n_hc, iters, split); + if (blockIdx.x == 0) { + for (int i = 0; i < mix_dim; ++i) { + dst[n_embd + i] = split[i]; + } + } + } + __syncthreads(); + + const int d = (int) blockIdx.x * blockDim.x + tid; + if (d < n_embd) { + float acc = 0.0f; + for (int h = 0; h < n_hc; ++h) { + acc += split[h] * hc_state[(size_t) h * n_embd + d]; + } + dst[d] = acc; + } +} + +static __global__ void ds4_hc_post_kernel( + const float * __restrict__ residual, + const float * __restrict__ block_out, + const float * __restrict__ split, + float * __restrict__ dst, + int n_embd, + int n_hc) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + const int total = n_embd * n_hc; + if (i >= total) { + return; + } + const int h = i / n_embd; + const int d = i - h * n_embd; + const float * post = split + n_hc; + const float * comb = split + 2 * n_hc; + float acc = block_out[d] * post[h]; + for (int src = 0; src < n_hc; ++src) { + acc += comb[h + src * n_hc] * residual[(size_t) src * n_embd + d]; + } + dst[i] = acc; +} + +static __global__ void ds4_hc_out_kernel( + const float * __restrict__ mix, + const float * __restrict__ base, + const float * __restrict__ hc_state, + float * __restrict__ dst, + int n_embd, + int n_hc, + float pre_scale) { + const int d = blockIdx.x * blockDim.x + threadIdx.x; + if (d >= n_embd) { + return; + } + float acc = 0.0f; + for (int h = 0; h < n_hc; ++h) { + const float wgt = ds4_hc_sigmoid(mix[h] * pre_scale + base[h]) + DS4_HC_SINKHORN_EPS; + acc += wgt * hc_state[(size_t) h * n_embd + d]; + } + dst[d] = acc; +} + +void ggml_cuda_op_ds4_hc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src2 = dst->src[2]; + + GGML_ASSERT(src0 && src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1 && src1->type == GGML_TYPE_F32); + GGML_ASSERT(src2 && src2->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int mode = ggml_get_op_params_i32(dst, 0); + const int n_embd = ggml_get_op_params_i32(dst, 1); + const int n_hc = ggml_get_op_params_i32(dst, 2); + + GGML_ASSERT(n_hc > 0 && n_hc <= DS4_HC_MAX_HC); + + cudaStream_t stream = ctx.stream(); + + switch (mode) { + case 0: { + const int iters = ggml_get_op_params_i32(dst, 3); + const float pre_scale = ggml_get_op_params_f32(dst, 4); + const float post_scale = ggml_get_op_params_f32(dst, 5); + const float comb_scale = ggml_get_op_params_f32(dst, 6); + const int pre_blocks = (n_embd + 255) / 256; + if (n_hc == 4) { + ds4_hc_pre_kernel_t<4><<>>( + (const float *) src0->data, (const float *) src1->data, + (const float *) src2->data, (float *) dst->data, + n_embd, iters, pre_scale, post_scale, comb_scale); + } else { + ds4_hc_pre_kernel<<>>( + (const float *) src0->data, (const float *) src1->data, + (const float *) src2->data, (float *) dst->data, + n_embd, n_hc, iters, pre_scale, post_scale, comb_scale); + } + } break; + case 1: { + const int total = n_embd * n_hc; + const int blocks = (total + 255) / 256; + ds4_hc_post_kernel<<>>( + (const float *) src0->data, (const float *) src1->data, + (const float *) src2->data, (float *) dst->data, + n_embd, n_hc); + } break; + case 2: { + const float pre_scale = ggml_get_op_params_f32(dst, 4); + const int blocks = (n_embd + 255) / 256; + ds4_hc_out_kernel<<>>( + (const float *) src0->data, (const float *) src1->data, + (const float *) src2->data, (float *) dst->data, + n_embd, n_hc, pre_scale); + } break; + default: + GGML_ABORT("ds4_hc: unknown mode"); + } +} diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cuh new file mode 100644 index 000000000..1a2cdf5eb --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ds4-hc.cuh @@ -0,0 +1,10 @@ +#pragma once + +#include "ggml-cuda/common.cuh" + +// Fused DeepSeek4 hyper-connection (HC) decode ops. See ds4-hc.cu for the +// per-mode contract (pre / post / out). Used by the opt-in +// DFLASH_DS4_FUSED_DECODE single-graph decode path; output is deterministic +// but not bit-identical to the CPU HC reference (expf ULP differences +// amplified by the sinkhorn iterations). +void ggml_cuda_op_ds4_hc(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/fattn.cu index 0eff4f1b7..e991bf4a3 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/fattn.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/fattn.cu @@ -7,6 +7,133 @@ #include "fattn-chunked.cuh" #include "fattn.cuh" +__device__ static float ds4_fa_block_sum(float v) { + __shared__ float smem[256]; + const int tid = threadIdx.x; + smem[tid] = v; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) smem[tid] += smem[tid + stride]; + __syncthreads(); + } + return smem[0]; +} + +__device__ static float ds4_fa_block_max(float v) { + __shared__ float smem[256]; + const int tid = threadIdx.x; + smem[tid] = v; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) smem[tid] = fmaxf(smem[tid], smem[tid + stride]); + __syncthreads(); + } + return smem[0]; +} + +__global__ static void ds4_flash_attn_d512_f32_shared_kv_kernel( + float * dst, + const float * q, + const float * k, + const float * v, + const float * sinks, + int n_tokens, + int n_heads, + int n_kv) { + constexpr int D = 512; + const int t = (int) blockIdx.x; + const int h = (int) blockIdx.y; + const int tid = (int) threadIdx.x; + if (t >= n_tokens || h >= n_heads) return; + + extern __shared__ float scores[]; + const float * qh = q + ((size_t) h * (size_t) n_tokens + (size_t) t) * D; + const float scale = rsqrtf((float) D); + + float local_max = sinks ? sinks[h] : -3.402823466e38f; + for (int r = tid; r < n_kv; r += blockDim.x) { + const float * kr = k + (size_t) r * D; + float dot = 0.0f; +#pragma unroll + for (int d = 0; d < D; ++d) { + dot += qh[d] * kr[d]; + } + const float s = dot * scale; + scores[r] = s; + local_max = fmaxf(local_max, s); + } + const float max_score = ds4_fa_block_max(local_max); + + float local_sum = 0.0f; + for (int r = tid; r < n_kv; r += blockDim.x) { + const float w = expf(scores[r] - max_score); + scores[r] = w; + local_sum += w; + } + if (tid == 0 && sinks) { + local_sum += expf(sinks[h] - max_score); + } + const float denom = ds4_fa_block_sum(local_sum); + const float inv_denom = 1.0f / denom; + + for (int d = tid; d < D; d += blockDim.x) { + float acc = 0.0f; + for (int r = 0; r < n_kv; ++r) { + acc += scores[r] * v[(size_t) r * D + d]; + } + dst[((size_t) t * (size_t) n_heads + (size_t) h) * D + d] = acc * inv_denom; + } +} + +static bool ggml_cuda_ds4_flash_attn_d512_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const char * e = getenv("DFLASH_DS4_FLASH_ATTN"); + if (!e || !e[0] || e[0] == (char)48) { + return false; + } + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + if (!Q || !K || !V || mask || + Q->type != GGML_TYPE_F32 || K->type != GGML_TYPE_F32 || V->type != GGML_TYPE_F32 || + dst->type != GGML_TYPE_F32 || + Q->ne[0] != 512 || K->ne[0] != 512 || V->ne[0] != 512 || + K->ne[2] != 1 || V->ne[2] != 1 || + Q->ne[3] != 1 || K->ne[3] != 1 || V->ne[3] != 1 || + dst->ne[0] != 512 || dst->ne[1] != Q->ne[2] || dst->ne[2] != Q->ne[1] || + Q->nb[0] != (int64_t) sizeof(float) || + K->nb[0] != (int64_t) sizeof(float) || + V->nb[0] != (int64_t) sizeof(float) || + dst->nb[0] != (int64_t) sizeof(float)) { + return false; + } + if (sinks && (sinks->type != GGML_TYPE_F32 || sinks->ne[0] != Q->ne[2])) { + return false; + } + + const int n_tokens = (int) Q->ne[1]; + const int n_heads = (int) Q->ne[2]; + const int n_kv = (int) K->ne[1]; + if (n_tokens <= 0 || n_heads <= 0 || n_kv <= 0) { + return false; + } + + cudaStream_t stream = ctx.stream(); + dim3 grid((unsigned) n_tokens, (unsigned) n_heads, 1); + const size_t shmem = (size_t) n_kv * sizeof(float); + ds4_flash_attn_d512_f32_shared_kv_kernel<<>>( + (float *) dst->data, + (const float *) Q->data, + (const float *) K->data, + (const float *) V->data, + sinks ? (const float *) sinks->data : nullptr, + n_tokens, + n_heads, + n_kv); + return true; +} + template static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; @@ -354,6 +481,10 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } } + static const bool ds4_allow_d512_nomask = [] { + const char * e = getenv("DFLASH_DS4_FLASH_ATTN"); + return e && e[0] && e[0] != (char)48; + }(); const int cc = ggml_cuda_info().devices[device].cc; @@ -374,7 +505,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const if (V->ne[0] != K->ne[0]) { return BEST_FATTN_KERNEL_NONE; } - if (!gqa_opt_applies) { + if (!gqa_opt_applies && !ds4_allow_d512_nomask) { return BEST_FATTN_KERNEL_NONE; } break; @@ -570,6 +701,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_set_device(ctx.device); + if (ggml_cuda_ds4_flash_attn_d512_f32(ctx, dst)) { + return; + } switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("fatal error"); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/getrows.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/getrows.cu index 1d7c6d17a..73a28e529 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/getrows.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/getrows.cu @@ -199,6 +199,30 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_Q4_0_ROCMFP4: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q2_0_ROCMFP2: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q3_0_ROCMFPX: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q6_0_ROCMFPX: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q8_0_ROCMFPX: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_TQ3_0: get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index feb9104a9..e5463faf9 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -64,6 +64,7 @@ #include "ggml-cuda/cumsum.cuh" #include "ggml-cuda/fill.cuh" #include "ggml-cuda/moe-fused.cuh" +#include "ggml-cuda/ds4-hc.cuh" #include "ggml.h" #include @@ -2361,7 +2362,12 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return false; } - static constexpr std::array valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI }; + static constexpr std::array valid_glu_ops = { + GGML_GLU_OP_SWIGLU, + GGML_GLU_OP_GEGLU, + GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_SWIGLU_DS4, + }; if (std::find(valid_glu_ops.begin(), valid_glu_ops.end(), ggml_get_glu_op(glu)) == valid_glu_ops.end()) { return false; @@ -2417,6 +2423,12 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { return use_mul_mat_vec_f; } +static inline void ggml_cuda_set_fusion_glu_params(ggml_cuda_mm_fusion_args_host & fusion_data, const ggml_tensor * glu) { + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_param0 = ggml_get_op_params_f32(glu, 2); + fusion_data.glu_param1 = ggml_get_op_params_f32(glu, 3); +} + static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; @@ -2850,6 +2862,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_GLU_OP_SWIGLU_OAI: ggml_cuda_op_swiglu_oai(ctx, dst); break; + case GGML_GLU_OP_SWIGLU_DS4: + ggml_cuda_op_swiglu_ds4(ctx, dst); + break; case GGML_GLU_OP_GEGLU_ERF: ggml_cuda_op_geglu_erf(ctx, dst); break; @@ -2869,6 +2884,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_MOE_FUSED: ggml_cuda_op_moe_fused(ctx, dst); break; + case GGML_OP_DS4_HC: + ggml_cuda_op_ds4_hc(ctx, dst); + break; case GGML_OP_GROUP_NORM: ggml_cuda_op_group_norm(ctx, dst); break; @@ -4071,7 +4089,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud fusion_data.gate = gate_n->src[0]; fusion_data.x_bias = up_bias_tensor; fusion_data.gate_bias = gate_bias_tensor; - fusion_data.glu_op = ggml_get_glu_op(glu); + ggml_cuda_set_fusion_glu_params(fusion_data, glu); ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data); fused_mul_mat_vec = true; @@ -4084,7 +4102,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud fusion_data.gate = gate_n->src[0]; fusion_data.x_bias = up_bias_tensor; fusion_data.gate_bias = gate_bias_tensor; - fusion_data.glu_op = ggml_get_glu_op(glu); + ggml_cuda_set_fusion_glu_params(fusion_data, glu); ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); fused_mul_mat_vec = true; @@ -4108,7 +4126,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) { ggml_cuda_mm_fusion_args_host fusion_data{}; fusion_data.gate = gate->src[0]; - fusion_data.glu_op = ggml_get_glu_op(glu); + ggml_cuda_set_fusion_glu_params(fusion_data, glu); ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data); fused_mul_mat_vec = true; @@ -4119,7 +4137,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud if (ggml_cuda_should_fuse_mul_mat_vec_q(up)) { ggml_cuda_mm_fusion_args_host fusion_data{}; fusion_data.gate = gate->src[0]; - fusion_data.glu_op = ggml_get_glu_op(glu); + ggml_cuda_set_fusion_glu_params(fusion_data, glu); ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); fused_mul_mat_vec = true; @@ -4987,6 +5005,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_GLU_OP_GEGLU: case GGML_GLU_OP_SWIGLU: case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_SWIGLU_DS4: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: return ggml_is_contiguous_1(op->src[0]); @@ -5000,6 +5019,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type); case GGML_OP_MOE_FUSED: return true; + case GGML_OP_DS4_HC: + return true; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { @@ -5045,6 +5066,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_Q4_0_ROCMFP4: + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + case GGML_TYPE_Q2_0_ROCMFP2: + case GGML_TYPE_Q3_0_ROCMFPX: + case GGML_TYPE_Q6_0_ROCMFPX: + case GGML_TYPE_Q8_0_ROCMFPX: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5080,6 +5107,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_0_ROCMFP4: + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + case GGML_TYPE_Q2_0_ROCMFP2: + case GGML_TYPE_Q3_0_ROCMFPX: + case GGML_TYPE_Q6_0_ROCMFPX: + case GGML_TYPE_Q8_0_ROCMFPX: case GGML_TYPE_TQ3_0: return true; default: @@ -5186,7 +5219,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_CONCAT: { ggml_type src0_type = op->src[0]->type; - return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; + return src0_type == GGML_TYPE_F32 || + src0_type == GGML_TYPE_F16 || + src0_type == GGML_TYPE_BF16 || + src0_type == GGML_TYPE_I8; } break; case GGML_OP_CONV_TRANSPOSE_1D: { diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvf.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvf.cu index d91472024..6e4db8e8b 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvf.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvf.cu @@ -51,6 +51,8 @@ static __global__ void mul_mat_vec_f( bool use_bias = false; bool use_gate_bias = false; ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU; + float glu_param0 = 0.0f; + float glu_param1 = 0.0f; const T * gate_x = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; @@ -60,6 +62,8 @@ static __global__ void mul_mat_vec_f( use_bias = fusion.x_bias != nullptr; use_gate_bias = fusion.gate_bias != nullptr; glu_op = fusion.glu_op; + glu_param0 = fusion.glu_param0; + glu_param1 = fusion.glu_param1; if (use_gate) { gate_x = static_cast(fusion.gate); @@ -357,7 +361,11 @@ static __global__ void mul_mat_vec_f( value *= ggml_cuda_op_gelu_single(gate_value); break; case GGML_GLU_OP_SWIGLU_OAI: { - value = ggml_cuda_op_swiglu_oai_single(gate_value, value); + value = ggml_cuda_op_swiglu_oai_single(gate_value, value, glu_param0, glu_param1); + break; + } + case GGML_GLU_OP_SWIGLU_DS4: { + value = ggml_cuda_op_swiglu_ds4_single(gate_value, value, glu_param0); break; } default: @@ -369,7 +377,7 @@ static __global__ void mul_mat_vec_f( dst[tid*stride_col_dst + row] = value; if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, gate_x, x_bias, gate_bias, sumf_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_param0, glu_param1, gate_x, x_bias, gate_bias, sumf_gate); } } @@ -668,6 +676,8 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor fusion_local.gate_bias = fusion->gate_bias->data; } fusion_local.glu_op = fusion->glu_op; + fusion_local.glu_param0 = fusion->glu_param0; + fusion_local.glu_param1 = fusion->glu_param1; } const int64_t s01 = src0->nb[1] / ts_src0; diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu index 78dcf500f..620a1b3fb 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu @@ -17,6 +17,12 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) case GGML_TYPE_Q8_0: return vec_dot_q8_0_q8_1; case GGML_TYPE_MXFP4: return vec_dot_mxfp4_q8_1; case GGML_TYPE_NVFP4: return vec_dot_nvfp4_q8_1; + case GGML_TYPE_Q4_0_ROCMFP4: return vec_dot_rocmfp4_q8_1; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: return vec_dot_rocmfp4_fast_q8_1; + case GGML_TYPE_Q2_0_ROCMFP2: return vec_dot_rocmfpx_fp2_q8_1; + case GGML_TYPE_Q3_0_ROCMFPX: return vec_dot_rocmfpx_fp3_q8_1; + case GGML_TYPE_Q6_0_ROCMFPX: return vec_dot_rocmfpx_fp6_q8_1; + case GGML_TYPE_Q8_0_ROCMFPX: return vec_dot_rocmfpx_fp8_q8_1; case GGML_TYPE_Q2_K: return vec_dot_q2_K_q8_1; case GGML_TYPE_Q3_K: return vec_dot_q3_K_q8_1; case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1; @@ -44,6 +50,12 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { case GGML_TYPE_Q8_0: return VDR_Q8_0_Q8_1_MMVQ; case GGML_TYPE_MXFP4: return VDR_MXFP4_Q8_1_MMVQ; case GGML_TYPE_NVFP4: return VDR_NVFP4_Q8_1_MMVQ; + case GGML_TYPE_Q4_0_ROCMFP4: return VDR_ROCMFP4_Q8_1_MMVQ; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: return VDR_ROCMFP4_FAST_Q8_1_MMVQ; + case GGML_TYPE_Q2_0_ROCMFP2: return VDR_ROCMFP2_Q8_1_MMVQ; + case GGML_TYPE_Q3_0_ROCMFPX: return VDR_ROCMFP3_Q8_1_MMVQ; + case GGML_TYPE_Q6_0_ROCMFPX: return VDR_ROCMFP6_Q8_1_MMVQ; + case GGML_TYPE_Q8_0_ROCMFPX: return VDR_ROCMFP8_Q8_1_MMVQ; case GGML_TYPE_Q2_K: return VDR_Q2_K_Q8_1_MMVQ; case GGML_TYPE_Q3_K: return VDR_Q3_K_Q8_1_MMVQ; case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ; @@ -454,6 +466,8 @@ static __global__ void mul_mat_vec_q( const float * x_bias = nullptr; const float * gate_bias = nullptr; ggml_glu_op active_glu; + float glu_param0 = 0.0f; + float glu_param1 = 0.0f; if constexpr (has_fusion) { use_gate = fusion.gate != nullptr; @@ -463,6 +477,8 @@ static __global__ void mul_mat_vec_q( x_bias = (const float *) fusion.x_bias; gate_bias = (const float *) fusion.gate_bias; active_glu = fusion.glu_op; + glu_param0 = fusion.glu_param0; + glu_param1 = fusion.glu_param1; } @@ -596,7 +612,11 @@ static __global__ void mul_mat_vec_q( result *= ggml_cuda_op_gelu_single(gate_value); break; case GGML_GLU_OP_SWIGLU_OAI: { - result = ggml_cuda_op_swiglu_oai_single(gate_value, result); + result = ggml_cuda_op_swiglu_oai_single(gate_value, result, glu_param0, glu_param1); + break; + } + case GGML_GLU_OP_SWIGLU_DS4: { + result = ggml_cuda_op_swiglu_ds4_single(gate_value, result, glu_param0); break; } default: @@ -610,7 +630,7 @@ static __global__ void mul_mat_vec_q( } if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, gate_bias, x_bias, tmp_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, glu_param0, glu_param1, gate_bias, x_bias, tmp_gate); } } @@ -660,6 +680,8 @@ static __global__ void mul_mat_vec_q_moe( const float * x_bias = nullptr; const float * gate_bias = nullptr; ggml_glu_op active_glu; + float glu_param0 = 0.0f; + float glu_param1 = 0.0f; if constexpr (has_fusion) { use_gate = fusion.gate != nullptr; @@ -669,6 +691,8 @@ static __global__ void mul_mat_vec_q_moe( x_bias = (const float *) fusion.x_bias; gate_bias = (const float *) fusion.gate_bias; active_glu = fusion.glu_op; + glu_param0 = fusion.glu_param0; + glu_param1 = fusion.glu_param1; } // partial sum for each thread @@ -721,7 +745,10 @@ static __global__ void mul_mat_vec_q_moe( result *= ggml_cuda_op_gelu_single(gate_value); break; case GGML_GLU_OP_SWIGLU_OAI: - result = ggml_cuda_op_swiglu_oai_single(gate_value, result); + result = ggml_cuda_op_swiglu_oai_single(gate_value, result, glu_param0, glu_param1); + break; + case GGML_GLU_OP_SWIGLU_DS4: + result = ggml_cuda_op_swiglu_ds4_single(gate_value, result, glu_param0); break; default: result = result * gate_value; @@ -733,7 +760,7 @@ static __global__ void mul_mat_vec_q_moe( } if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, vgate, x_bias, gate_bias, active_glu, tmp_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, vgate, x_bias, gate_bias, active_glu, glu_param0, glu_param1, tmp_gate); } } @@ -1082,6 +1109,42 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_Q4_0_ROCMFP4: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_Q4_0_ROCMFP4_FAST: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_Q2_0_ROCMFP2: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_Q3_0_ROCMFPX: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_Q6_0_ROCMFPX: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_Q8_0_ROCMFPX: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; case GGML_TYPE_Q2_K: mul_mat_vec_q_switch_ncols_dst (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, @@ -1219,6 +1282,8 @@ void ggml_cuda_mul_mat_vec_q( fusion_local.gate_bias = fusion->gate_bias->data; } fusion_local.glu_op = fusion->glu_op; + fusion_local.glu_param0 = fusion->glu_param0; + fusion_local.glu_param1 = fusion->glu_param1; } // If src0 is a temporary compute buffer, clear any potential padding. diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cu index 4ad30fa1f..ed7f3d02a 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cu @@ -338,6 +338,63 @@ void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_op_unary_gated(ctx, dst); } +// swiglu_ds4 + +template +static __global__ void swiglu_ds4_kernel(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, float limit) { + const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + + const int64_t j0 = (i / n) * o0 + (i % n); + const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + + const float gate_v = gate[j0]; + const float up_v = up[j1]; + + dst[i] = ggml_cuda_op_swiglu_ds4_single(gate_v, up_v, limit); +} + +template +static void swiglu_ds4_cuda(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, const float limit, cudaStream_t stream) { + const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE; + swiglu_ds4_kernel<<>>(gate, up, dst, k, n, o0, o1, limit); +} + +void ggml_cuda_op_swiglu_ds4(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + void * src0_d = src0->data; + void * src1_d = src1 ? src1->data : src0->data; + const int64_t src0_o = src0->nb[1]; + const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + void * dst_d = dst->data; + const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(src0->nb[0] == ggml_element_size(src0)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0)); + + GGML_ASSERT(src1); + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src1->nb[0] == ggml_element_size(src1)); + GGML_ASSERT(src1->ne[0] == nc); + GGML_ASSERT(src0->type == src1->type); + + const float limit = ggml_get_op_params_f32(dst, 2); + + swiglu_ds4_cuda((float *) src0_d, (float *) src1_d, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), limit, stream); +} + void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_op_unary_gated(ctx, dst); } diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cuh index f1dd2183a..1864ce2dc 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/unary.cuh @@ -81,6 +81,8 @@ void ggml_cuda_op_geglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +void ggml_cuda_op_swiglu_ds4(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst); @@ -110,3 +112,11 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float g out_glu = out_glu * (1.0f + g); return out_glu; } + +__device__ __forceinline__ float ggml_cuda_op_swiglu_ds4_single(float gate, float up, float limit) { + gate = fminf(gate, limit); + up = fmaxf(fminf(up, limit), -limit); + + const float silu = gate / (1.0f + expf(-gate)); + return silu * up; +} diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh index 40b2b41e7..8e8e79eb2 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh @@ -1,6 +1,7 @@ #pragma once #include "common.cuh" +#include "../../rocmfp4/rocmfp4_hip_codebook.cuh" #include @@ -322,6 +323,380 @@ static __device__ __forceinline__ float vec_dot_mxfp4_q8_1( return d * sumi; } +// === ROCMFP vec_dot device functions (ported from ROCmFPX) === +#ifndef GGML_ROCMFP4_Q8_1_MMQ_VDR +#define GGML_ROCMFP4_Q8_1_MMQ_VDR 8 +#endif + +#ifndef GGML_ROCMFP4_FAST_Q8_1_MMQ_VDR +#define GGML_ROCMFP4_FAST_Q8_1_MMQ_VDR GGML_ROCMFP4_Q8_1_MMQ_VDR +#endif + +#ifndef GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR +#define GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR 2 +#endif + +#if GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR != 1 && \ + GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR != 2 && \ + GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR != 4 +#error "GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR must be 1, 2, or 4" +#endif + +#define VDR_ROCMFP4_Q8_1_MMVQ 4 +#define VDR_ROCMFP4_Q8_1_MMQ GGML_ROCMFP4_Q8_1_MMQ_VDR +#define VDR_ROCMFP4_FAST_Q8_1_MMVQ GGML_ROCMFP4_FAST_Q8_1_MMVQ_VDR +#define VDR_ROCMFP4_FAST_Q8_1_MMQ GGML_ROCMFP4_FAST_Q8_1_MMQ_VDR +#define VDR_ROCMFP2_Q8_1_MMVQ 1 +#define VDR_ROCMFP3_Q8_1_MMVQ 2 +#ifndef VDR_ROCMFP6_Q8_1_MMVQ +#define VDR_ROCMFP6_Q8_1_MMVQ 4 +#endif +#ifndef GGML_ROCMFP6_FAST_SIGNMAG_PACK +#define GGML_ROCMFP6_FAST_SIGNMAG_PACK 0 +#endif +#ifndef GGML_ROCMFP6_MMVQ_HALF_BLOCK_SPLIT +// Enabled by default: hoisting the half-block scale selection out of the FP6 +// MMVQ dot loop measured ~1.8% faster tg on gfx1151 (Qwen3-0.6B Q6_0_ROCMFPX, +// 211.8 -> 215.7 t/s) with bit-identical results. Set to 0 to restore the +// per-element branch. +#define GGML_ROCMFP6_MMVQ_HALF_BLOCK_SPLIT 1 +#endif +#define VDR_ROCMFP8_Q8_1_MMVQ 2 + +#define VDR_ROCMFP2_Q8_1_MMQ 4 +#define VDR_ROCMFP3_Q8_1_MMQ 4 +#ifndef VDR_ROCMFP6_Q8_1_MMQ +#define VDR_ROCMFP6_Q8_1_MMQ 4 +#endif +#define VDR_ROCMFP8_Q8_1_MMQ 8 + +static __device__ __forceinline__ uint32_t rocmfpx_get_bits_vec_cuda(const uint8_t * src, const int bit_pos, const int nbits) { + uint32_t code = 0; + +#pragma unroll + for (int bit = 0; bit < nbits; ++bit) { + const int src_bit = bit_pos + bit; + code |= ((uint32_t) ((src[src_bit >> 3] >> (src_bit & 7)) & 1u)) << bit; + } + + return code; +} + +static __device__ __forceinline__ int rocmfpx_decode_fp3_code_vec_cuda(const uint32_t code) { + const uint32_t mag_code = code & 3u; + const int mag = mag_code == 3u ? 4 : (int) mag_code; + return (code & 4u) ? -mag : mag; +} + +static __device__ __forceinline__ int rocmfpx_decode_fp2_code_vec_cuda(const uint32_t code) { + switch (code & 3u) { + case 0: return ROCMFP2_KVALUE_0_I8; + case 1: return ROCMFP2_KVALUE_1_I8; + case 2: return ROCMFP2_KVALUE_2_I8; + default: return ROCMFP2_KVALUE_3_I8; + } +} + +static __device__ __forceinline__ int rocmfpx_decode_fp6_code_vec_cuda(const uint32_t code) { +#if GGML_ROCMFP6_FAST_SIGNMAG_PACK + const int mag = (int) (code & 31u); + const int sign = -((int) ((code >> 5) & 1u)); + return (mag ^ sign) - sign; +#else + const int mag = (int) (code & 31u); + return (code & 32u) ? -mag : mag; +#endif +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp6_bits24_vec_cuda(const uint32_t bits24) { + const char4 v = make_char4( + (int8_t) rocmfpx_decode_fp6_code_vec_cuda(bits24 & 63u), + (int8_t) rocmfpx_decode_fp6_code_vec_cuda((bits24 >> 6) & 63u), + (int8_t) rocmfpx_decode_fp6_code_vec_cuda((bits24 >> 12) & 63u), + (int8_t) rocmfpx_decode_fp6_code_vec_cuda((bits24 >> 18) & 63u)); + return *((const int *) &v); +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp3_vec_cuda(const uint8_t * qs, const int base) { + const char4 v = make_char4( + (int8_t) rocmfpx_decode_fp3_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 0)*3, 3)), + (int8_t) rocmfpx_decode_fp3_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 1)*3, 3)), + (int8_t) rocmfpx_decode_fp3_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 2)*3, 3)), + (int8_t) rocmfpx_decode_fp3_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 3)*3, 3))); + return *((const int *) &v); +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp2_bits8_vec_cuda(const uint32_t bits8) { +#if defined(GGML_USE_HIP) + const uint32_t values = + ((uint32_t) (uint8_t) (int8_t) ROCMFP2_KVALUE_0_I8) | + ((uint32_t) (uint8_t) (int8_t) ROCMFP2_KVALUE_1_I8 << 8) | + ((uint32_t) (uint8_t) (int8_t) ROCMFP2_KVALUE_2_I8 << 16) | + ((uint32_t) (uint8_t) (int8_t) ROCMFP2_KVALUE_3_I8 << 24); + const uint32_t selectors = + ((bits8 >> 0) & 3u) | + (((bits8 >> 2) & 3u) << 8) | + (((bits8 >> 4) & 3u) << 16) | + (((bits8 >> 6) & 3u) << 24); + return (int) __builtin_amdgcn_perm(0, values, selectors); +#else + const char4 v = make_char4( + (int8_t) rocmfpx_decode_fp2_code_vec_cuda((bits8 >> 0) & 3u), + (int8_t) rocmfpx_decode_fp2_code_vec_cuda((bits8 >> 2) & 3u), + (int8_t) rocmfpx_decode_fp2_code_vec_cuda((bits8 >> 4) & 3u), + (int8_t) rocmfpx_decode_fp2_code_vec_cuda((bits8 >> 6) & 3u)); + return *((const int *) &v); +#endif +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp2_vec_cuda(const uint8_t * qs, const int base) { + return rocmfpx_pack4_fp2_bits8_vec_cuda(qs[base >> 2]); +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp6_vec_cuda(const uint8_t * qs, const int base) { +#if GGML_ROCMFP6_FAST_SIGNMAG_PACK + uint32_t qs0, qs1, qs2, qs3, qs4, qs5; + memcpy(&qs0, qs + 0, 4); + memcpy(&qs1, qs + 4, 4); + memcpy(&qs2, qs + 8, 4); + memcpy(&qs3, qs + 12, 4); + memcpy(&qs4, qs + 16, 4); + memcpy(&qs5, qs + 20, 4); + + const uint32_t words[7] = { qs0, qs1, qs2, qs3, qs4, qs5, 0 }; + const int start_bit = 6 * base; + const int reg_idx = start_bit >> 5; + const int reg_shift = start_bit & 31; + const uint32_t val_low = words[reg_idx]; + const uint32_t val_high = words[reg_idx + 1]; + const uint32_t bits24 = (reg_shift == 0) ? (val_low & 0xFFFFFFu) : + (((val_low >> reg_shift) | (val_high << (32 - reg_shift))) & 0xFFFFFFu); + + return rocmfpx_pack4_fp6_bits24_vec_cuda(bits24); +#else + const char4 v = make_char4( + (int8_t) rocmfpx_decode_fp6_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 0)*6, 6)), + (int8_t) rocmfpx_decode_fp6_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 1)*6, 6)), + (int8_t) rocmfpx_decode_fp6_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 2)*6, 6)), + (int8_t) rocmfpx_decode_fp6_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 3)*6, 6))); + return *((const int *) &v); +#endif +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp6_expanded_vec_cuda(const int8_t * qs, const int base) { + const char4 v = make_char4(qs[base + 0], qs[base + 1], qs[base + 2], qs[base + 3]); + return *((const int *) &v); +} + +static __device__ __forceinline__ int rocmfpx_pack4_fp6_device_vec_cuda(const block_rocmfp6_device * bq6, const int base) { +#if GGML_ROCMFP6_EXPANDED_DEVICE + return rocmfpx_pack4_fp6_expanded_vec_cuda(bq6->qs, base); +#else + return rocmfpx_pack4_fp6_vec_cuda(bq6->qs, base); +#endif +} + +static __device__ __forceinline__ float vec_dot_rocmfp4_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_rocmfp4 * bq4 = (const block_rocmfp4 *) vbq + kbx; + + const int * q8 = (const int *) bq8_1->qs + iqs; + + int sumi0 = 0; + int sumi1 = 0; +#pragma unroll + for (int l = 0; l < VDR_ROCMFP4_Q8_1_MMVQ; ++l) { + const int aux_q4 = rocmfp4_get_qs_i32(bq4->qs, iqs + l); + const int2 v = rocmfp4_get_int_from_codebook_16(aux_q4, kvalues_rocmfp4); + + sumi0 = ggml_cuda_dp4a(v.x, q8[l + 0], sumi0); + sumi1 = ggml_cuda_dp4a(v.y, q8[l + 4], sumi1); + } + + const float db = __low2float(bq8_1->ds); + return db * (rocmfp4_ue4m3_to_fp32_half_finite(bq4->e[0]) * sumi0 + rocmfp4_ue4m3_to_fp32_half_finite(bq4->e[1]) * sumi1); +} + +static __device__ __forceinline__ float vec_dot_rocmfp4_fast_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_rocmfp4_fast * bq4 = (const block_rocmfp4_fast *) vbq + kbx; + + const int * q8 = (const int *) bq8_1->qs + iqs; + + int sumi = 0; +#pragma unroll + for (int l = 0; l < VDR_ROCMFP4_FAST_Q8_1_MMVQ; ++l) { + const int aux_q4 = rocmfp4_get_qs_i32(bq4->qs, iqs + l); + const int2 v = rocmfp4_get_int_from_codebook_16(aux_q4, kvalues_rocmfp4); + + sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi); + sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi); + } + + return __low2float(bq8_1->ds) * rocmfp4_ue4m3_to_fp32_half_finite(bq4->e) * sumi; +} + +static __device__ __forceinline__ float vec_dot_rocmfpx_fp2_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + // int32-consistent MMVQ layout: QI_ROCMFP2=2 (8B qs = 2 int32), VDR=1. + // iqs in {0,1} selects one 16-weight half-block: qs bytes [4*iqs .. 4*iqs+3] + scale e[iqs]. + // qs is a 10-byte-strided, byte-aligned array -> read byte-wise (uint8_t index); NEVER cast qs to int*/int2*. + const block_rocmfp2 * bq2 = (const block_rocmfp2 *) vbq + kbx; + + int sumi = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int val_packed = rocmfpx_pack4_fp2_bits8_vec_cuda((uint32_t) bq2->qs[4*iqs + j]); + const int u = get_int_b4(bq8_1->qs, 4*iqs + j); + sumi = ggml_cuda_dp4a(val_packed, u, sumi); + } + + const float db = __low2float(bq8_1->ds); + return db * rocmfpx_ue4m3_to_fp32_finite(bq2->e[iqs]) * sumi; +} + +static __device__ __forceinline__ float vec_dot_rocmfpx_fp3_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_rocmfp3 * bq3 = (const block_rocmfp3 *) vbq + kbx; + + uint32_t qs0, qs1, qs2; + memcpy(&qs0, bq3->qs + 0, 4); + memcpy(&qs1, bq3->qs + 4, 4); + memcpy(&qs2, bq3->qs + 8, 4); + + const uint32_t qs[4] = { qs0, qs1, qs2, 0 }; + + int sumi0 = 0; + int sumi1 = 0; + + // The two half-block scales (e[0]/e[1]) split at element QK_ROCMFP3/2. base + // < QK_ROCMFP3/2 is equivalent to (iqs+i) < QK_ROCMFP3/8, so for a VDR + // window that lies entirely in one half the accumulator choice is loop + // invariant and can be hoisted out of the unrolled loop. A straddling window + // (only possible for VDRs that cross the midpoint) still uses the exact + // per-element branch, so results are bit-identical either way. + const bool fp3_first_half = iqs + VDR_ROCMFP3_Q8_1_MMVQ <= QK_ROCMFP3/8; + const bool fp3_second_half = iqs >= QK_ROCMFP3/8; + +#pragma unroll + for (int i = 0; i < VDR_ROCMFP3_Q8_1_MMVQ; ++i) { + const int base = 4 * (iqs + i); + const int start_bit = 12 * (iqs + i); + const int reg_idx = start_bit >> 5; + const int reg_shift = start_bit & 31; + const uint32_t val_low = qs[reg_idx]; + const uint32_t val_high = qs[reg_idx + 1]; + const uint32_t bits12 = (reg_shift == 0) ? (val_low & 0xFFFu) : (((val_low >> reg_shift) | (val_high << (32 - reg_shift))) & 0xFFFu); + + const char4 v = make_char4( + (int8_t) rocmfpx_decode_fp3_code_vec_cuda(bits12 & 7u), + (int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 3) & 7u), + (int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 6) & 7u), + (int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 9) & 7u)); + const int val_packed = *((const int *) &v); + + const int u = get_int_b4(bq8_1->qs, iqs + i); + + if (fp3_first_half) { + sumi0 = ggml_cuda_dp4a(val_packed, u, sumi0); + } else if (fp3_second_half) { + sumi1 = ggml_cuda_dp4a(val_packed, u, sumi1); + } else if (base < QK_ROCMFP3/2) { + sumi0 = ggml_cuda_dp4a(val_packed, u, sumi0); + } else { + sumi1 = ggml_cuda_dp4a(val_packed, u, sumi1); + } + } + + const float db = __low2float(bq8_1->ds); + return db * (rocmfpx_ue4m3_to_fp32_finite(bq3->e[0]) * sumi0 + rocmfpx_ue4m3_to_fp32_finite(bq3->e[1]) * sumi1); +} + +static __device__ __forceinline__ float vec_dot_rocmfpx_fp6_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_rocmfp6_device * bq6 = (const block_rocmfp6_device *) vbq + kbx; + +#if !GGML_ROCMFP6_EXPANDED_DEVICE + uint32_t qs0, qs1, qs2, qs3, qs4, qs5; + memcpy(&qs0, bq6->qs + 0, 4); + memcpy(&qs1, bq6->qs + 4, 4); + memcpy(&qs2, bq6->qs + 8, 4); + memcpy(&qs3, bq6->qs + 12, 4); + memcpy(&qs4, bq6->qs + 16, 4); + memcpy(&qs5, bq6->qs + 20, 4); + + // Trailing 0 pad: the last FP6 window reads qs[reg_idx + 1] with reg_idx==5; + // its high bits are always masked out, so 0 keeps the result bit-identical + // while avoiding a stack over-read (matches the FP3 sibling below). + const uint32_t qs[7] = { qs0, qs1, qs2, qs3, qs4, qs5, 0 }; +#endif + + int sumi0 = 0; + int sumi1 = 0; + +#if GGML_ROCMFP6_MMVQ_HALF_BLOCK_SPLIT + const bool fp6_first_half = iqs + VDR_ROCMFP6_Q8_1_MMVQ <= QK_ROCMFP6/8; + const bool fp6_second_half = iqs >= QK_ROCMFP6/8; +#endif + +#pragma unroll + for (int i = 0; i < VDR_ROCMFP6_Q8_1_MMVQ; ++i) { + const int base = 4 * (iqs + i); +#if GGML_ROCMFP6_EXPANDED_DEVICE + const int val_packed = rocmfpx_pack4_fp6_device_vec_cuda(bq6, base); +#else + const int start_bit = 6 * base; + const int reg_idx = start_bit >> 5; + const int reg_shift = start_bit & 31; + const uint32_t val_low = qs[reg_idx]; + const uint32_t val_high = qs[reg_idx + 1]; + const uint32_t bits24 = (reg_shift == 0) ? (val_low & 0xFFFFFFu) : + (((val_low >> reg_shift) | (val_high << (32 - reg_shift))) & 0xFFFFFFu); + + const int val_packed = rocmfpx_pack4_fp6_bits24_vec_cuda(bits24); +#endif + const int u = get_int_b4(bq8_1->qs, iqs + i); + +#if GGML_ROCMFP6_MMVQ_HALF_BLOCK_SPLIT + if (fp6_first_half) { + sumi0 = ggml_cuda_dp4a(val_packed, u, sumi0); + } else if (fp6_second_half) { + sumi1 = ggml_cuda_dp4a(val_packed, u, sumi1); + } else +#endif + if (base < QK_ROCMFP6/2) { + sumi0 = ggml_cuda_dp4a(val_packed, u, sumi0); + } else { + sumi1 = ggml_cuda_dp4a(val_packed, u, sumi1); + } + } + + const float db = __low2float(bq8_1->ds); + return db * (rocmfpx_ue4m3_to_fp32_finite(bq6->e[0]) * sumi0 + rocmfpx_ue4m3_to_fp32_finite(bq6->e[1]) * sumi1); +} + +static __device__ __forceinline__ float vec_dot_rocmfpx_fp8_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_rocmfp8 * bq8 = (const block_rocmfp8 *) vbq + kbx; + + int v[VDR_ROCMFP8_Q8_1_MMVQ]; + int u[VDR_ROCMFP8_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_ROCMFP8_Q8_1_MMVQ; ++i) { + v[i] = get_int_b1(bq8->qs, iqs + i); + u[i] = get_int_b4(bq8_1->qs, iqs + i); + } + + return vec_dot_q8_0_q8_1_impl( + v, u, rocmfpx_ue4m3_to_fp32_finite(bq8->e), __low2half(bq8_1->ds)); +} + #define VDR_NVFP4_Q8_1_MMVQ 4 #define VDR_NVFP4_Q8_1_MMQ 8 diff --git a/server/deps/llama.cpp/ggml/src/ggml.c b/server/deps/llama.cpp/ggml/src/ggml.c index fda42ef76..591ab9bda 100644 --- a/server/deps/llama.cpp/ggml/src/ggml.c +++ b/server/deps/llama.cpp/ggml/src/ggml.c @@ -9,6 +9,8 @@ // FIXME: required here for quantization functions #include "ggml-quants.h" +#include "../rocmfp4/rocmfp4.h" +#include "../rocmfpx/rocmfpx.h" #ifdef GGML_USE_CPU_HBM #include @@ -719,6 +721,54 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_q8_0, .from_float_ref = (ggml_from_float_t) quantize_row_q8_0_ref, }, + [GGML_TYPE_Q4_0_ROCMFP4] = { + .type_name = "q4_0_rocmfp4", + .blck_size = QK_ROCMFP4, + .type_size = sizeof(block_rocmfp4), + .is_quantized = true, + .to_float = (ggml_to_float_t) rocmfp4_dequantize_row_q4_0, + .from_float_ref = (ggml_from_float_t) rocmfp4_quantize_row_q4_0_ref, + }, + [GGML_TYPE_Q4_0_ROCMFP4_FAST] = { + .type_name = "q4_0_rocmfp4_fast", + .blck_size = QK_ROCMFP4, + .type_size = sizeof(block_rocmfp4_fast), + .is_quantized = true, + .to_float = (ggml_to_float_t) rocmfp4_dequantize_row_q4_0_fast, + .from_float_ref = (ggml_from_float_t) rocmfp4_quantize_row_q4_0_fast_ref, + }, + [GGML_TYPE_Q3_0_ROCMFPX] = { + .type_name = "q3_0_rocmfpx", + .blck_size = QK_ROCMFP3, + .type_size = sizeof(block_rocmfp3), + .is_quantized = true, + .to_float = (ggml_to_float_t) rocmfpx_dequantize_row_fp3, + .from_float_ref = (ggml_from_float_t) rocmfpx_quantize_row_fp3_ref, + }, + [GGML_TYPE_Q2_0_ROCMFP2] = { + .type_name = "q2_0_rocmfp2", + .blck_size = QK_ROCMFP2, + .type_size = sizeof(block_rocmfp2), + .is_quantized = true, + .to_float = (ggml_to_float_t) rocmfpx_dequantize_row_fp2, + .from_float_ref = (ggml_from_float_t) rocmfpx_quantize_row_fp2_ref, + }, + [GGML_TYPE_Q6_0_ROCMFPX] = { + .type_name = "q6_0_rocmfpx", + .blck_size = QK_ROCMFP6, + .type_size = sizeof(block_rocmfp6), + .is_quantized = true, + .to_float = (ggml_to_float_t) rocmfpx_dequantize_row_fp6, + .from_float_ref = (ggml_from_float_t) rocmfpx_quantize_row_fp6_ref, + }, + [GGML_TYPE_Q8_0_ROCMFPX] = { + .type_name = "q8_0_rocmfpx", + .blck_size = QK_ROCMFP8, + .type_size = sizeof(block_rocmfp8), + .is_quantized = true, + .to_float = (ggml_to_float_t) rocmfpx_dequantize_row_fp8, + .from_float_ref = (ggml_from_float_t) rocmfpx_quantize_row_fp8_ref, + }, [GGML_TYPE_Q8_1] = { .type_name = "q8_1", .blck_size = QK8_1, @@ -1074,9 +1124,13 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "GLU", "TURBO_WHT", + + "MOE_FUSED", + + "DS4_HC", }; -static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); +static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1187,9 +1241,13 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "glu(x)", "turbo_wht(a)", + + "moe_fused(x)", + + "ds4_hc(x)", }; -static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); +static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -1225,11 +1283,12 @@ static const char * GGML_GLU_OP_NAME[GGML_GLU_OP_COUNT] = { "GEGLU", "SWIGLU", "SWIGLU_OAI", + "SWIGLU_DS4", "GEGLU_ERF", "GEGLU_QUICK", }; -static_assert(GGML_GLU_OP_COUNT == 6, "GGML_GLU_OP_COUNT != 6"); +static_assert(GGML_GLU_OP_COUNT == 7, "GGML_GLU_OP_COUNT != 7"); static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN"); @@ -1428,6 +1487,20 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; + // ROCmFPX presets are mixed-quant recipes (see ggml/rocmfp4/README.md); + // report the dominant transformer-tensor type. + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4: + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_LEAN: + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_COHERENT: wtype = GGML_TYPE_Q4_0_ROCMFP4; break; + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_FAST: + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_FAST_COHERENT: + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_STRIX: + case GGML_FTYPE_MOSTLY_Q4_0_ROCMFP4_STRIX_LEAN: wtype = GGML_TYPE_Q4_0_ROCMFP4_FAST; break; + case GGML_FTYPE_MOSTLY_Q6_0_ROCMFPX: wtype = GGML_TYPE_Q6_0_ROCMFPX; break; + case GGML_FTYPE_MOSTLY_Q8_0_ROCMFPX: wtype = GGML_TYPE_Q8_0_ROCMFPX; break; + case GGML_FTYPE_MOSTLY_Q3_0_ROCMFPX: wtype = GGML_TYPE_Q3_0_ROCMFPX; break; + case GGML_FTYPE_MOSTLY_Q2_0_ROCMFP2: + case GGML_FTYPE_MOSTLY_Q2_0_ROCMFP2_STRIX: wtype = GGML_TYPE_Q2_0_ROCMFP2; break; } GGML_ASSERT(wtype != GGML_TYPE_COUNT); @@ -3025,6 +3098,16 @@ struct ggml_tensor * ggml_swiglu_split( return ggml_glu_impl(ctx, a, b, GGML_GLU_OP_SWIGLU, false); } +struct ggml_tensor * ggml_swiglu_ds4_split( + struct ggml_context * ctx, + struct ggml_tensor * gate, + struct ggml_tensor * up, + float clamp) { + struct ggml_tensor * result = ggml_glu_impl(ctx, gate, up, GGML_GLU_OP_SWIGLU_DS4, false); + ggml_set_op_params_f32(result, 2, clamp); + return result; +} + // ggml_geglu_erf struct ggml_tensor * ggml_geglu_erf( @@ -7991,3 +8074,93 @@ struct ggml_tensor * ggml_laguna_moe_combine( return result; } + +struct ggml_tensor * ggml_ds4_hc_pre( + struct ggml_context * ctx, + struct ggml_tensor * mix, + struct ggml_tensor * base, + struct ggml_tensor * hc_state, + int n_hc, + int sinkhorn_iters, + float pre_scale, + float post_scale, + float comb_scale) { + GGML_ASSERT(mix->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(hc_state->type == GGML_TYPE_F32); + GGML_ASSERT(n_hc > 0 && n_hc <= 8); + const int64_t mix_dim = 2*(int64_t)n_hc + (int64_t)n_hc*n_hc; + GGML_ASSERT(ggml_nelements(mix) == mix_dim); + GGML_ASSERT(ggml_nelements(base) >= mix_dim); + GGML_ASSERT(ggml_nelements(hc_state) % n_hc == 0); + const int64_t n_embd = ggml_nelements(hc_state) / n_hc; + + struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd + mix_dim); + result->op = GGML_OP_DS4_HC; + result->src[0] = mix; + result->src[1] = base; + result->src[2] = hc_state; + ggml_set_op_params_i32(result, 0, 0); + ggml_set_op_params_i32(result, 1, (int32_t) n_embd); + ggml_set_op_params_i32(result, 2, (int32_t) n_hc); + ggml_set_op_params_i32(result, 3, (int32_t) sinkhorn_iters); + ggml_set_op_params_f32(result, 4, pre_scale); + ggml_set_op_params_f32(result, 5, post_scale); + ggml_set_op_params_f32(result, 6, comb_scale); + return result; +} + +struct ggml_tensor * ggml_ds4_hc_post( + struct ggml_context * ctx, + struct ggml_tensor * residual_hc, + struct ggml_tensor * block_out, + struct ggml_tensor * split, + int n_hc) { + GGML_ASSERT(residual_hc->type == GGML_TYPE_F32); + GGML_ASSERT(block_out->type == GGML_TYPE_F32); + GGML_ASSERT(split->type == GGML_TYPE_F32); + GGML_ASSERT(n_hc > 0 && n_hc <= 8); + const int64_t mix_dim = 2*(int64_t)n_hc + (int64_t)n_hc*n_hc; + GGML_ASSERT(ggml_nelements(split) == mix_dim); + GGML_ASSERT(ggml_nelements(residual_hc) % n_hc == 0); + const int64_t n_embd = ggml_nelements(residual_hc) / n_hc; + GGML_ASSERT(ggml_nelements(block_out) == n_embd); + + struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (int64_t) n_embd * n_hc); + result->op = GGML_OP_DS4_HC; + result->src[0] = residual_hc; + result->src[1] = block_out; + result->src[2] = split; + ggml_set_op_params_i32(result, 0, 1); + ggml_set_op_params_i32(result, 1, (int32_t) n_embd); + ggml_set_op_params_i32(result, 2, (int32_t) n_hc); + return result; +} + +struct ggml_tensor * ggml_ds4_hc_out( + struct ggml_context * ctx, + struct ggml_tensor * mix, + struct ggml_tensor * base, + struct ggml_tensor * hc_state, + int n_hc, + float pre_scale) { + GGML_ASSERT(mix->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(hc_state->type == GGML_TYPE_F32); + GGML_ASSERT(n_hc > 0 && n_hc <= 8); + GGML_ASSERT(ggml_nelements(mix) >= n_hc); + GGML_ASSERT(ggml_nelements(base) >= n_hc); + GGML_ASSERT(ggml_nelements(hc_state) % n_hc == 0); + const int64_t n_embd = ggml_nelements(hc_state) / n_hc; + + struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + result->op = GGML_OP_DS4_HC; + result->src[0] = mix; + result->src[1] = base; + result->src[2] = hc_state; + ggml_set_op_params_i32(result, 0, 2); + ggml_set_op_params_i32(result, 1, (int32_t) n_embd); + ggml_set_op_params_i32(result, 2, (int32_t) n_hc); + ggml_set_op_params_f32(result, 4, pre_scale); + return result; +} From a5f1146f49d3a1b218db0b6a06a79631a099798b Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:50:22 +0200 Subject: [PATCH 2/2] fix(ggml): make ROCmFP4 codebook self-contained on CUDA The non-HIP fallback of rocmfp4_get_int_from_codebook_16 / rocmfp4_get_low_int_from_codebook_16 called get_int_from_table_16, which is defined in vecdotq.cuh. TUs that pull in this header without vecdotq.cuh (fattn-chunked.cu reaches it via the fattn dequant chain) failed to compile under nvcc: rocmfp4_hip_codebook.cuh: error: identifier "get_int_from_table_16" is undefined The HIP path never hit this (it uses __builtin_amdgcn_perm), so the ROCm CI and the Strix build stayed green while the sm_86 CUDA build broke. Fix: inline the generic table expander (the generic branch of get_int_from_table_16, verbatim) as a static helper in this header, so the fallback no longer depends on include order. Bit-identical; the HIP hot path is unchanged. --- .../ggml/rocmfp4/rocmfp4_hip_codebook.cuh | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh index 1d9106112..b5dea2e11 100644 --- a/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh +++ b/server/deps/llama.cpp/ggml/rocmfp4/rocmfp4_hip_codebook.cuh @@ -9,6 +9,28 @@ #define GGML_ROCMFP4_UNALIGNED_QS_DWORD_LOAD 1 #endif +#if !defined(GGML_USE_HIP) +// Self-contained 16-entry table expander for the non-HIP fallback. This header +// is pulled into translation units that do not include vecdotq.cuh (where the +// generic get_int_from_table_16 lives) - e.g. fattn-chunked.cu - so relying on +// that symbol breaks the CUDA build on include order. This is the generic +// branch of get_int_from_table_16 verbatim, so results are bit-identical; only +// the HIP path (Strix) uses __builtin_amdgcn_perm and never reaches here. +static __device__ __forceinline__ int2 rocmfp4_table16_fallback(const int & q4, const int8_t * table) { + const int q0_32 = (q4 >> 0) & 0x0F0F0F0F; + const int8_t * q0_8 = (const int8_t *) &q0_32; + const char4 val0_8 = make_char4( + table[q0_8[0]], table[q0_8[1]], table[q0_8[2]], table[q0_8[3]]); + + const int q1_32 = (q4 >> 4) & 0x0F0F0F0F; + const int8_t * q1_8 = (const int8_t *) &q1_32; + const char4 val1_8 = make_char4( + table[q1_8[0]], table[q1_8[1]], table[q1_8[2]], table[q1_8[3]]); + + return make_int2(*((const int *) &val0_8), *((const int *) &val1_8)); +} +#endif + static __device__ __forceinline__ int rocmfp4_get_qs_i32(const void * x, const int & i32) { #if defined(GGML_USE_HIP) && GGML_ROCMFP4_UNALIGNED_QS_DWORD_LOAD return *((const int *) ((const uint8_t *) x + 4*i32)); @@ -52,7 +74,7 @@ static __device__ __forceinline__ int2 rocmfp4_get_int_from_codebook_16(const in __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even), __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd)); #else - return get_int_from_table_16(q4, fallback_table); + return rocmfp4_table16_fallback(q4, fallback_table); #endif } @@ -74,6 +96,6 @@ static __device__ __forceinline__ int rocmfp4_get_low_int_from_codebook_16(const return __builtin_amdgcn_perm(v_high, v_low, mask); #else - return get_int_from_table_16(q4, fallback_table).x; + return rocmfp4_table16_fallback(q4, fallback_table).x; #endif }