diff --git a/conversion/__init__.py b/conversion/__init__.py index 2c38123dff8d..7f34f85f5c6a 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -41,6 +41,8 @@ "CodeShellForCausalLM": "codeshell", "CogVLMForCausalLM": "cogvlm", "Cohere2ForCausalLM": "command_r", + "Cohere2MoeForCausalLM": "command_r", + "Cohere2VisionForConditionalGeneration": "command_r", "CohereForCausalLM": "command_r", "DbrxForCausalLM": "dbrx", "DeciLMForCausalLM": "deci", diff --git a/conversion/base.py b/conversion/base.py index 8e12af6c5dd5..7b26654e08ac 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1416,6 +1416,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1": # ref: https://huggingface.co/CohereLabs/tiny-aya-base res = "tiny_aya" + if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e": + # ref: https://huggingface.co/CohereLabs/command-a-plus-05-2026 + res = "tiny_aya" if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" diff --git a/conversion/command_r.py b/conversion/command_r.py index 603288d165c5..626a3db55fe8 100644 --- a/conversion/command_r.py +++ b/conversion/command_r.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import Iterable, TYPE_CHECKING +import re + +from typing import Callable, Iterable, TYPE_CHECKING import torch @@ -55,3 +57,77 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Cohere2MoeForCausalLM") +@ModelBase.register("Cohere2VisionForConditionalGeneration") +class Cohere2MoeModel(TextModel): + model_arch = gguf.MODEL_ARCH.COHERE2_MOE + + # accumulated per-expert weights before merging into a 3D tensor + _experts: list[dict[str, Tensor]] | None = None + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + # base class merges text_config into hparams, so all keys are at the top level + p = self.hparams + self.gguf_writer.add_logit_scale(p.get("logit_scale", 1.0)) + self.gguf_writer.add_sliding_window(p["sliding_window"]) + self.gguf_writer.add_vocab_size(p["vocab_size"]) + # expert_count and expert_used_count are already set by the base class + self.gguf_writer.add_expert_shared_count(p["num_shared_experts"]) + self.gguf_writer.add_expert_feed_forward_length(p["intermediate_size"]) + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + self.gguf_writer.add_expert_weights_norm(p.get("norm_topk_prob", False)) + + # head_dim is explicit in the config (128); hidden_size/n_heads = 4096/128 = 32 would be wrong + rotary_pct = p.get("rotary_pct", 1.0) + head_dim = p.get("head_dim", p["hidden_size"] // p["num_attention_heads"]) + self.gguf_writer.add_rope_dimension_count(int(rotary_pct * head_dim)) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + swa_pattern = p.get("layer_switch", p.get("_sliding_window_pattern", 4)) + self.gguf_writer.add_sliding_window_pattern(swa_pattern) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + # apply base class filtering first (strips "language_model." prefix) + result = super().filter_tensors(item) + if result is None: + return None + name, gen = result + # skip vision encoder and multimodal projector tensors for text-only GGUF + if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")): + return None + return name, gen + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # skip zero bias tensors (all biases in Cohere2 are zero-valued) + if name.endswith(".bias"): + if torch.any(data_torch != 0): + raise ValueError(f"Bias tensor {name!r} is not zero.") + return + + # experts are stored as 128 individual 2D tensors; merge them into one 3D tensor + if re.search(r'\.mlp\.experts\.\d+\.', name): + n_experts = self.hparams["num_experts"] + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas: list[Tensor] = [] + for xid in range(n_experts): + ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[bid].pop(ename)) + merged = torch.stack(datas, dim=0) + merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight" + yield from super().modify_tensors(merged, merged_name, bid) + return + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 7fdcf03d7d11..296add71d62a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -442,6 +442,7 @@ class MODEL_ARCH(IntEnum): XVERSE = auto() COMMAND_R = auto() COHERE2 = auto() + COHERE2_MOE = auto() DBRX = auto() OLMO = auto() OLMO2 = auto() @@ -957,6 +958,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.XVERSE: "xverse", MODEL_ARCH.COMMAND_R: "command-r", MODEL_ARCH.COHERE2: "cohere2", + MODEL_ARCH.COHERE2_MOE: "cohere2-moe", MODEL_ARCH.DBRX: "dbrx", MODEL_ARCH.OLMO: "olmo", MODEL_ARCH.OLMO2: "olmo2", @@ -2735,6 +2737,22 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.COHERE2_MOE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + ], MODEL_ARCH.DBRX: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index c9eead18aa39..d599183a3f9d 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -66,6 +66,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_XVERSE, "xverse" }, { LLM_ARCH_COMMAND_R, "command-r" }, { LLM_ARCH_COHERE2, "cohere2" }, + { LLM_ARCH_COHERE2_MOE, "cohere2-moe" }, { LLM_ARCH_DBRX, "dbrx" }, { LLM_ARCH_OLMO, "olmo" }, { LLM_ARCH_OLMO2, "olmo2" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 89cf16cc37cf..287779905ddb 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -70,6 +70,7 @@ enum llm_arch { LLM_ARCH_XVERSE, LLM_ARCH_COMMAND_R, LLM_ARCH_COHERE2, + LLM_ARCH_COHERE2_MOE, LLM_ARCH_DBRX, LLM_ARCH_OLMO, LLM_ARCH_OLMO2, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 528e4c9c069f..db7fc9e153ec 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -21,6 +21,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_GEMMA3: case LLM_ARCH_GEMMA3N: case LLM_ARCH_COHERE2: + case LLM_ARCH_COHERE2_MOE: case LLM_ARCH_OLMO2: case LLM_ARCH_BITNET: case LLM_ARCH_T5: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8bf20a716eba..4cef76ce9173 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -150,6 +150,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_command_r(params); case LLM_ARCH_COHERE2: return new llama_model_cohere2(params); + case LLM_ARCH_COHERE2_MOE: + return new llama_model_cohere2_moe(params); case LLM_ARCH_DBRX: return new llama_model_dbrx(params); case LLM_ARCH_OLMO: @@ -2253,6 +2255,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_XVERSE: case LLM_ARCH_COMMAND_R: case LLM_ARCH_COHERE2: + case LLM_ARCH_COHERE2_MOE: case LLM_ARCH_OLMO: case LLM_ARCH_ARCTIC: case LLM_ARCH_DEEPSEEK: diff --git a/src/models/cohere2_moe.cpp b/src/models/cohere2_moe.cpp new file mode 100644 index 000000000000..bd2837e8288e --- /dev/null +++ b/src/models/cohere2_moe.cpp @@ -0,0 +1,183 @@ +#include "models.h" + +void llama_model_cohere2_moe::load_arch_hparams(llama_model_loader & ml) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + uint32_t swa_period = 4; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); + hparams.set_swa_pattern(swa_period); + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + } + + switch (hparams.n_layer) { + case 32: type = LLM_TYPE_UNKNOWN; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_cohere2_moe::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_expert_shared = hparams.n_expert_shared; + const int64_t n_ff_exp = hparams.n_ff_exp; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }, 0); + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); + + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_exp * n_expert_shared }, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd }, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_exp * n_expert_shared }, 0); + } +} + +std::unique_ptr llama_model_cohere2_moe::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_cohere2_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + const int64_t n_expert_shared = hparams.n_expert_shared; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + const float f_logit_scale = hparams.f_logit_scale; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * inp_pos = build_inp_pos(); + auto * inp_attn = build_attn_inp_kv_iswa(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + const bool is_swa = hparams.is_swa(il); + + // pre-norm (shared for parallel attention + FFN) + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); + cb(cur, "attn_norm", il); + ggml_tensor * ffn_inp = cur; + + // self-attention + { + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); + + if (is_swa) { + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, + 1.0f / sqrtf(float(n_embd_head)), il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids); + } + + ggml_tensor * attn_out = cur; + + // MoE FFN (parallel with attention, same pre-norm input) + { + ggml_tensor * moe_out = build_moe_ffn(ffn_inp, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + model.layers[il].ffn_gate_up_exps); + cb(moe_out, "ffn_moe_out", il); + + // shared experts (averaged) + ggml_tensor * ffn_shexp = build_ffn(ffn_inp, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + if (n_expert_shared > 1) { + ffn_shexp = ggml_scale(ctx0, ffn_shexp, 1.0f / n_expert_shared); + cb(ffn_shexp, "ffn_shexp_avg", il); + } + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + + // parallel block: residual + attn + ffn + cur = ggml_add(ctx0, cur, inpL); + cur = ggml_add(ctx0, cur, attn_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + + if (f_logit_scale) { + cur = ggml_scale(ctx0, cur, f_logit_scale); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 7e551eb965b1..fc08932f8461 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -899,6 +899,19 @@ struct llama_model_cohere2 : public llama_model_base { }; +struct llama_model_cohere2_moe : public llama_model_base { + llama_model_cohere2_moe(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_dbrx : public llama_model_base { llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override;