From 88133d2983da8f9ca260ae48c823a43f1df3cd8a Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Mon, 9 Feb 2026 01:00:54 +0100 Subject: [PATCH] Migrate Qwen3.5 to IMROPE --- src/llama-model.cpp | 5 +++-- src/models/qwen3-5.cpp | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8fc61aee3727..931a88e08bb7 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2418,6 +2418,7 @@ void llama_model::load_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); // Load linear attention (gated delta net) parameters ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); @@ -8755,8 +8756,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_PANGU_EMBED: case LLM_ARCH_AFMOE: case LLM_ARCH_QWEN3NEXT: - case LLM_ARCH_QWEN3_5: - case LLM_ARCH_QWEN3_5_MOE: case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: return LLAMA_ROPE_TYPE_NEOX; @@ -8765,6 +8764,8 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { return LLAMA_ROPE_TYPE_MROPE; case LLM_ARCH_QWEN3VL: case LLM_ARCH_QWEN3VLMOE: + case LLM_ARCH_QWEN3_5: + case LLM_ARCH_QWEN3_5_MOE: return LLAMA_ROPE_TYPE_IMROPE; case LLM_ARCH_GLM4: diff --git a/src/models/qwen3-5.cpp b/src/models/qwen3-5.cpp index 0947299d73a7..bf7a863282cb 100644 --- a/src/models/qwen3-5.cpp +++ b/src/models/qwen3-5.cpp @@ -132,15 +132,18 @@ ggml_tensor * llm_build_qwen3_5::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext( + int sections[4]; + std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); + + Qcur = ggml_rope_multi( ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( + Kcur = ggml_rope_multi( ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il);