diff --git a/conversion/base.py b/conversion/base.py index 08fd3747c408..cf991d370941 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1154,7 +1154,7 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca or "projector." in name or "pre_mm_projector_norm" in name \ or "image_newline" in name or "view_seperator" in name \ or "patch_embed" in name or "patch_embedding" in name \ - or "patch_merger." in name or "model.connector." in name: + or "patch_merger." in name or "patch_merge_mlp." in name or "model.connector." in name: return None return super().filter_tensors(item) @@ -1201,7 +1201,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_embedding_length(n_embd) logger.info(f"gguf: embedding length = {n_embd}") - if (n_ff := self.find_hparam(["prefix_dense_intermediate_size", "intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None: + if (n_ff := self.find_hparam(["prefix_dense_intermediate_size", "dense_intermediate_size", "intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None: self.gguf_writer.add_feed_forward_length(n_ff) logger.info(f"gguf: feed forward length = {n_ff}") diff --git a/conversion/minimax.py b/conversion/minimax.py index bce703dcabd6..d3cb5cf94076 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -62,14 +62,9 @@ class MiniMaxM3Model(TextModel): _experts_cache: dict[int, dict[str, Tensor]] = {} def set_gguf_parameters(self): - # dense layers use dense_intermediate_size, experts use intermediate_size. Base - # writes feed_forward_length from intermediate_size, so swap in the dense width - # and emit the expert width separately. - expert_ff = self.find_hparam(["intermediate_size"]) - self.hparams["intermediate_size"] = self.find_hparam(["dense_intermediate_size"]) super().set_gguf_parameters() - self.gguf_writer.add_expert_feed_forward_length(expert_ff) + self.gguf_writer.add_expert_feed_forward_length(self.find_hparam(["intermediate_size"])) self.gguf_writer.add_rope_dimension_count(self.find_hparam(["rotary_dim"])) self.gguf_writer.add_expert_shared_count(self.find_hparam(["n_shared_experts"])) self.gguf_writer.add_expert_weights_scale(self.find_hparam(["routed_scaling_factor"])) @@ -105,14 +100,6 @@ def set_gguf_parameters(self): self.gguf_writer.add_leading_dense_block_count(n_dense) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): - # text-only: drop vision, projector, patch-merge tensors - if name.startswith(("vision_tower", "multi_modal_projector", "patch_merge_mlp")): - return - - # strip VL wrapper prefix to match tensor_mapping names - if name.startswith("language_model."): - name = name[len("language_model."):] - # Gemma-style (1 + w) RMSNorm: bake the +1 in so llama.cpp can use plain RMSNorm if name.endswith("norm.weight"): data_torch = data_torch + 1.0 diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1d7e88388b15..2f061b37d9e4 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1408,6 +1408,17 @@ ggml_tensor * llm_graph_context::build_ffn( cur = ggml_swiglu(ctx0, cur); cb(cur, "ffn_swiglu", il); } break; + case LLM_FFN_SWIGLU_OAI: + if (gate && type_gate == LLM_FFN_PAR) { + //Same constants as LLM_FFN_SWIGLU_OAI_MOE + const float alpha = 1.702f; + const float limit = 7.0f; + cur = ggml_swiglu_oai(ctx0, cur, tmp, alpha, limit); + cb(cur, "ffn_swiglu_oai", il); + type_gate = LLM_FFN_SEQ; + } else { + GGML_ABORT("LLM_FFN_SWIGLU_OAI requires a parallel gate"); + } break; case LLM_FFN_GEGLU: { cur = ggml_geglu(ctx0, cur); diff --git a/src/llama-graph.h b/src/llama-graph.h index 0fcb4c6ecfc6..621c8d5859ba 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -45,6 +45,7 @@ enum llm_ffn_op_type : int { LLM_FFN_SWIGLU, LLM_FFN_GEGLU, LLM_FFN_REGLU, + LLM_FFN_SWIGLU_OAI, LLM_FFN_SWIGLU_OAI_MOE, }; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index ef2625e46586..1c5e54fe9883 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -799,6 +799,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_122B_A10B: return "122B.A10B"; case LLM_TYPE_196B_A11B: return "196B.A11B"; case LLM_TYPE_230B_A10B: return "230B.A10B"; + case LLM_TYPE_428B_A23B: return "428B.A23B"; case LLM_TYPE_235B_A22B: return "235B.A22B"; case LLM_TYPE_300B_A47B: return "300B.A47B"; case LLM_TYPE_310B_A15B: return "310B.A15B"; diff --git a/src/llama-model.h b/src/llama-model.h index a2d470efee6d..15739c579d24 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -133,6 +133,7 @@ enum llm_type { LLM_TYPE_122B_A10B, // Qwen3.5 LLM_TYPE_196B_A11B, // Step3.5-Flash LLM_TYPE_230B_A10B, // Minimax M2 + LLM_TYPE_428B_A23B, // Minimax M3 LLM_TYPE_235B_A22B, LLM_TYPE_300B_A47B, // Ernie MoE big LLM_TYPE_310B_A15B, // /MiMo-V2-Flash diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 1073831e020d..b1e7e0f396aa 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -16,7 +16,7 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); - ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head, false); ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size, false); ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k, false); @@ -24,7 +24,10 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks, false); msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks }; - type = LLM_TYPE_UNKNOWN; + switch (hparams.n_layer()) { + case 60: type = LLM_TYPE_428B_A23B; break; + default: type = LLM_TYPE_UNKNOWN; + } } void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) { @@ -389,10 +392,6 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); // partial rotary: head_dim != n_rot, so don't assert n_embd_head == n_rot - // swigluoai params, shared by dense and expert FFNs - const float swiglu_alpha = 1.702f; - const float swiglu_limit = 7.0f; - ggml_tensor * cur; ggml_tensor * inpL; @@ -452,16 +451,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); // per-head QK RMSNorm (weights already include Gemma's +1) Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); @@ -574,10 +565,12 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ if ((uint32_t) il < hparams.n_layer_dense_lead) { // leading dense FFN (swigluoai) - ggml_tensor * g = build_lora_mm(model.layers[il].ffn_gate, cur); - ggml_tensor * u = build_lora_mm(model.layers[il].ffn_up, cur); - g = ggml_swiglu_oai(ctx0, g, u, swiglu_alpha, swiglu_limit); - cur = build_lora_mm(model.layers[il].ffn_down, g); + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); } else { // routed experts (swigluoai MoE) @@ -595,10 +588,12 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ cb(moe_out, "ffn_moe_out", il); // shared expert (swigluoai) - ggml_tensor * sg = build_lora_mm(model.layers[il].ffn_gate_shexp, cur); - ggml_tensor * su = build_lora_mm(model.layers[il].ffn_up_shexp, cur); - sg = ggml_swiglu_oai(ctx0, sg, su, swiglu_alpha, swiglu_limit); - ggml_tensor * ffn_shexp = build_lora_mm(model.layers[il].ffn_down_shexp, sg); + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il); cb(ffn_shexp, "ffn_shexp", il); cur = ggml_add(ctx0, moe_out, ffn_shexp);