From 2e53b96fee574e6bd8822da38cd13a11e0f54c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 16:42:23 +0100 Subject: [PATCH 01/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 201 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index a4de237e03..40a99c0036 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -3948,6 +3948,207 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): return [(self.map_tensor_name(name), data_torch)] return super().modify_tensors(data_torch, name, bid) +@ModelBase.register("Glm4MoeForCausalLM") +class Glm4MoeModel(TextModel): + model_arch = gguf.MODEL_ARCH.GLM4_MOE + + def set_vocab(self): + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained( + self.dir_model, trust_remote_code=True + ) + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + tokens, toktypes, tokpre = self.get_vocab_base() + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab._set_special_token( + "eos", tokenizer.get_added_vocab()["<|endoftext|>"] + ) + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) + special_vocab._set_special_token("eog", tokenizer.get_added_vocab()["<|user|>"]) + special_vocab._set_special_token("eog", tokenizer.get_added_vocab()["<|observation|>"]) + special_vocab._set_special_token( + "unk", tokenizer.get_added_vocab()["<|endoftext|>"] + ) + special_vocab._set_special_token( + "bos", tokenizer.get_added_vocab()["<|endoftext|>"] + ) + special_vocab.add_to_gguf(self.gguf_writer) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + if (rope_dim := self.hparams.get("head_dim")) is None: + rope_dim = ( + self.hparams["hidden_size"] // self.hparams["num_attention_heads"] + ) + self.gguf_writer.add_rope_dimension_count( + int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)) + ) + + # MoE parameters + if (n_experts := self.hparams.get("n_routed_experts")) is not None: + self.gguf_writer.add_expert_count(n_experts) + # Note: expert_used_count is already set by parent class using num_experts_per_tok + if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None: + self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size) + if (n_shared_experts := self.hparams.get("n_shared_experts")) is not None: + self.gguf_writer.add_expert_shared_count(n_shared_experts) + if (first_k_dense_replace := self.hparams.get("first_k_dense_replace")) is not None: + self.gguf_writer.add_leading_dense_block_count(first_k_dense_replace) + + # Expert gating function (sigmoid for GLM4_MOE) + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + + # Routed scaling factor + if (routed_scaling_factor := self.hparams.get("routed_scaling_factor")) is not None: + self.gguf_writer.add_expert_weights_scale(routed_scaling_factor) + + # Normalise topk probabilities + if (norm_topk_prob := self.hparams.get("norm_topk_prob")) is not None: + self.gguf_writer.add_expert_weights_norm(norm_topk_prob) + + _experts: list[dict[str, Tensor]] | None = None + _shared_experts: list[dict[str, Tensor]] | None = None + + def modify_tensors( + self, data_torch: Tensor, name: str, bid: int | None + ) -> Iterable[tuple[str, Tensor]]: + if name.startswith("model.visual."): # ignore visual part + return [] + elif name.startswith("model.language_model."): + name = name.replace("language_model.", "") # for multimodal variants + + # Handle main token embedding + if name == "model.embed_tokens.weight": + return [(self.map_tensor_name("token_embd.weight"), data_torch)] + + # Handle routed experts + if name.find("mlp.experts") != -1 and "shared_experts" not in name: + n_experts = self.hparams["n_routed_experts"] + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + # Extend experts array if needed (for models where actual layers > num_hidden_layers) + while len(self._experts) <= bid: + self._experts.append({}) + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + tensors: list[tuple[str, Tensor]] = [] + + # merge the experts into a single 3d tensor + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas: list[Tensor] = [] + + for xid in range(n_experts): + ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[bid][ename]) + del self._experts[bid][ename] + + data_torch = torch.stack(datas, dim=0) + # Generate GGUF tensor names for merged experts + if w_name == "down_proj": + new_name = f"blk.{bid}.ffn_down_exps.weight" + elif w_name == "gate_proj": + new_name = f"blk.{bid}.ffn_gate_exps.weight" + elif w_name == "up_proj": + new_name = f"blk.{bid}.ffn_up_exps.weight" + else: + merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight" + new_name = self.map_tensor_name(merged_name) + tensors.append((new_name, data_torch)) + return tensors + else: + return [] + + # Handle expert gating input (routing gate) + if ".mlp.gate.e_score_correction_bias" in name: + new_name = name.replace("model.layers.", "blk.").replace( + ".mlp.gate.e_score_correction_bias", ".ffn_gate_inp.bias" + ) + return [(new_name, data_torch)] + elif ".mlp.gate.weight" in name: + new_name = name.replace("model.layers.", "blk.").replace( + ".mlp.gate.weight", ".ffn_gate_inp.weight" + ) + return [(new_name, data_torch)] + + # Handle shared expert tensors + if ".mlp.shared_experts." in name: + new_name = name.replace("model.layers.", "blk.").replace(".mlp.shared_experts.", ".ffn_") + if "gate_proj" in new_name: + new_name = new_name.replace("gate_proj", "gate_shexp") + elif "down_proj" in new_name: + new_name = new_name.replace("down_proj", "down_shexp") + elif "up_proj" in new_name: + new_name = new_name.replace("up_proj", "up_shexp") + return [(new_name, data_torch)] + + # Handle regular dense FFN layers (for hybrid dense/MoE architecture) + if ".mlp." in name and "experts" not in name and "_shexp" not in name: + if "gate_proj" in name: + new_name = name.replace("model.layers.", "blk.").replace( + ".mlp.gate_proj.weight", ".ffn_gate.weight" + ) + elif "up_proj" in name: + new_name = name.replace("model.layers.", "blk.").replace( + ".mlp.up_proj.weight", ".ffn_up.weight" + ) + elif "down_proj" in name: + new_name = name.replace("model.layers.", "blk.").replace( + ".mlp.down_proj.weight", ".ffn_down.weight" + ) + else: + new_name = name + return [(self.map_tensor_name(new_name), data_torch)] + + # Handle special NextN tensors - preserve for future MTP support + if ( + ".embed_tokens." in name + or ".shared_head." in name + or ".eh_proj." in name + or ".enorm." in name + or ".hnorm." in name + ): + new_name = name.replace("model.layers.", "blk.").replace("model.", "").replace(".weight", "") + return [(new_name, data_torch)] + + # GLM tensor mapping - handle directly without map_tensor_name + if ".input_layernorm." in name: + new_name = name.replace("model.layers.", "blk.").replace(".input_layernorm.", ".attn_norm.") + return [(new_name, data_torch)] + elif ".post_attention_layernorm." in name: + new_name = name.replace("model.layers.", "blk.").replace(".post_attention_layernorm.", ".ffn_norm.") + return [(new_name, data_torch)] + elif ".self_attn." in name: + # Map GLM self_attn to standard attention naming + new_name = name.replace("model.layers.", "blk.").replace(".self_attn.", ".attn_") + if "q_proj" in new_name: + new_name = new_name.replace("q_proj", "q") + elif "k_proj" in new_name: + new_name = new_name.replace("k_proj", "k") + elif "v_proj" in new_name: + new_name = new_name.replace("v_proj", "v") + elif "o_proj" in new_name: + new_name = new_name.replace("o_proj", "output") + return [(new_name, data_torch)] + + return super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + if self._experts is not None: + # flatten `list[dict[str, Tensor]]` into `list[str]` + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}") @Model.register("ChatGLMModel", "ChatGLMForConditionalGeneration") class ChatGLMModel(Model): From 89ebd61768f62a3213973416440bd5b54fe393c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 16:47:13 +0100 Subject: [PATCH 02/55] Update constants.py --- gguf-py/gguf/constants.py | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 32a667e262..92722dc32d 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -220,6 +220,7 @@ class MODEL_ARCH(IntEnum): OPENELM = auto() ARCTIC = auto() DEEPSEEK2 = auto() + GLM4_MOE = auto() CHATGLM = auto() BITNET = auto() BITNET_25 = auto() @@ -262,6 +263,9 @@ class MODEL_TENSOR(IntEnum): FFN_GATE_EXP = auto() FFN_DOWN_EXP = auto() FFN_UP_EXP = auto() + FFN_GATE_EXPS = auto() # merged experts + FFN_DOWN_EXPS = auto() # merged experts + FFN_UP_EXPS = auto() # merged experts FFN_GATE_SHEXP = auto() FFN_DOWN_SHEXP = auto() FFN_UP_SHEXP = auto() @@ -314,6 +318,12 @@ class MODEL_TENSOR(IntEnum): ENC_FFN_DOWN = auto() ENC_FFN_UP = auto() ENC_OUTPUT_NORM = auto() + NEXTN_EH_PROJ = auto() # nextn tensors (glm4moe) + NEXTN_EMBED_TOKENS = auto() # nextn tensors (glm4moe) + NEXTN_ENORM = auto() # nextn tensors (glm4moe) + NEXTN_HNORM = auto() # nextn tensors (glm4moe) + NEXTN_SHARED_HEAD_HEAD = auto() # nextn tensors (glm4moe) + NEXTN_SHARED_HEAD_NORM = auto() # nextn tensors (glm4moe) MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { @@ -358,6 +368,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.ARCTIC: "arctic", MODEL_ARCH.DEEPSEEK2: "deepseek2", MODEL_ARCH.CHATGLM: "chatglm", + MODEL_ARCH.GLM4_MOE: "glm4moe", MODEL_ARCH.BITNET: "bitnet", MODEL_ARCH.BITNET_25: "bitnet-25", MODEL_ARCH.T5: "t5", @@ -404,6 +415,9 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate_exps", MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps", MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps", + MODEL_TENSOR.FFN_GATE_EXPS: "blk.{bid}.ffn_gate_exps", # merged experts + MODEL_TENSOR.FFN_DOWN_EXPS: "blk.{bid}.ffn_down_exps", # merged experts + MODEL_TENSOR.FFN_UP_EXPS: "blk.{bid}.ffn_up_exps", # merged experts MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b", MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm", MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in", @@ -451,6 +465,13 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ENC_FFN_DOWN: "enc.blk.{bid}.ffn_down", MODEL_TENSOR.ENC_FFN_UP: "enc.blk.{bid}.ffn_up", MODEL_TENSOR.ENC_OUTPUT_NORM: "enc.output_norm", + # NextN/MTP tensors (GLM4_MOE) + MODEL_TENSOR.NEXTN_EH_PROJ: "blk.{bid}.eh_proj", + MODEL_TENSOR.NEXTN_EMBED_TOKENS: "blk.{bid}.embed_tokens", + MODEL_TENSOR.NEXTN_ENORM: "blk.{bid}.enorm", + MODEL_TENSOR.NEXTN_HNORM: "blk.{bid}.hnorm", + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.shared_head.head", + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.shared_head.norm", } MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { @@ -1070,6 +1091,36 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.GLM4_MOE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, # dense layers + MODEL_TENSOR.FFN_DOWN, # dense layers + MODEL_TENSOR.FFN_UP, # dense layers + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXPS, + MODEL_TENSOR.FFN_DOWN_EXPS, + MODEL_TENSOR.FFN_UP_EXPS, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + # NextN/MTP tensors - preserved but unused + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, + ], MODEL_ARCH.BITNET: [ MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, From f619be787150b3ef0e7a535b409e47c6807582ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 18:55:25 +0100 Subject: [PATCH 03/55] Update llama.cpp --- src/llama.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 4f4428df8f..05760f14cc 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -284,6 +284,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_DEEPSEEK2, "deepseek2" }, { LLM_ARCH_CHATGLM, "chatglm" }, { LLM_ARCH_GLM4, "glm4" }, + { LLM_ARCH_GLM4_MOE, "glm4moe" }, { LLM_ARCH_BITNET, "bitnet" }, { LLM_ARCH_BITNET_25, "bitnet-25" }, { LLM_ARCH_BITNET_B158, "bitnet-b1.58" }, @@ -1407,6 +1408,39 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" }, }, }, + { + LLM_ARCH_GLM4_MOE, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" }, + { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, // dense layers + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, // dense layers + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, // dense layers + { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" }, + { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" }, + { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" }, + { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" }, + { LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" }, + { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" }, + { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" }, + // NextN/MTP tensors - preserved but unused + { LLM_TENSOR_NEXTN_EH_PROJ, "blk.%d.eh_proj" }, + { LLM_TENSOR_NEXTN_EMBED_TOKENS, "blk.%d.embed_tokens" }, + { LLM_TENSOR_NEXTN_ENORM, "blk.%d.enorm" }, + { LLM_TENSOR_NEXTN_HNORM, "blk.%d.hnorm" }, + { LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "blk.%d.shared_head.head" }, + { LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "blk.%d.shared_head.norm" }, + }, + }, { LLM_ARCH_BITNET, { From 662283ffb3ab7e2f2742982e6c513e097b622913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 18:57:36 +0100 Subject: [PATCH 04/55] Update llama.cpp --- src/llama.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 05760f14cc..47e8673896 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -226,6 +226,7 @@ enum llm_arch { LLM_ARCH_DEEPSEEK2, LLM_ARCH_CHATGLM, LLM_ARCH_GLM4, + LLM_ARCH_GLM4_MOE, LLM_ARCH_BITNET, LLM_ARCH_BITNET_25, LLM_ARCH_BITNET_B158, @@ -610,6 +611,12 @@ enum llm_tensor { LLM_TENSOR_ENC_FFN_DOWN, LLM_TENSOR_ENC_FFN_UP, LLM_TENSOR_ENC_OUTPUT_NORM, + LLM_TENSOR_NEXTN_EH_PROJ, + LLM_TENSOR_NEXTN_EMBED_TOKENS, + LLM_TENSOR_NEXTN_ENORM, + LLM_TENSOR_NEXTN_HNORM, + LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, + LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, }; static const std::map> LLM_TENSOR_NAMES = { From 0c419310f3b953b4c7cf2dacfa2f93b55a074482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:02:28 +0100 Subject: [PATCH 05/55] Update llama.cpp --- src/llama.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 47e8673896..ceddf11149 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5333,6 +5333,8 @@ static const char * llama_model_type_name(e_model type) { case MODEL_70B: return "70B"; case MODEL_142B: return "142B"; case MODEL_236B: return "236B"; + case MODEL_106B_A12B: return "106B.A12B"; + case MODEL_355B_A32B: return "355B.A32B"; case MODEL_314B: return "314B"; case MODEL_405B: return "405B"; case MODEL_671B: return "671B"; From 56f5b23b2318f176349699548f12d6d34365f250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:06:48 +0100 Subject: [PATCH 06/55] Update llama.cpp --- src/llama.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index ceddf11149..5d5381bd16 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6090,6 +6090,29 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_GLM4_MOE: + { + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + // MoE parameters + ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, 0); + ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, 0); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, 0); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, 0); + + // Expert gating function (GLM4_MOE uses sigmoid) + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + } + + switch (hparams.n_layer) { + case 47: model.type = e_model::MODEL_106B_A12B; break; // GLM-4.5-Air (46 layers + 1 NextN layer) + case 93: model.type = e_model::MODEL_355B_A32B; break; // GLM-4.5 (92 layers + 1 NextN layer) + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; case LLM_ARCH_BITNET: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); From 8dc1c03a4ff6ca9f541154d741779be8ab348627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:07:50 +0100 Subject: [PATCH 07/55] Update llama.cpp --- src/llama.cpp | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 5d5381bd16..3e6e76911b 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9013,6 +9013,91 @@ static bool llm_load_tensors( } } } break; + case LLM_ARCH_GLM4_MOE: + { + const int64_t n_expert = hparams.n_expert; + const int64_t n_expert_used = hparams.n_expert_used; + const int64_t n_expert_shared = hparams.n_expert_shared; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); + + // GLM-style attention with bias terms + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); + layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, TENSOR_NOT_REQUIRED); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); + + // K/Q norm tensors (optional for GLM-4.5 355B variant) + layer.attn_q_norm = create_tensor( + tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED); + layer.attn_k_norm = create_tensor( + tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); + + // Check if this layer uses MoE or dense FFN based on n_layer_dense_lead + const bool use_moe = + (hparams.n_expert > 0) && (static_cast(i) >= hparams.n_layer_dense_lead); + + if (use_moe) { + // MoE layers + layer.ffn_gate_inp = + create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); + + if (n_expert == 0) { + GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers"); + } + if (n_expert_used == 0) { + GGML_ASSERT(hparams.n_expert_used > 0 && + "n_expert_used must be > 0 for GLM4_MOE MoE layers"); + } + + // MoE branch + const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + + layer.ffn_gate_exps = create_tensor( + tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0); + layer.ffn_down_exps = create_tensor( + tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0); + layer.ffn_up_exps = create_tensor( + tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0); + + // Shared expert + if (n_expert_shared > 0) { + const int64_t n_ff_shexp = n_ff_exp * n_expert_shared; + layer.ffn_gate_shexp = create_tensor( + tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); + layer.ffn_down_shexp = create_tensor( + tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_shexp, n_embd }, 0); + layer.ffn_up_shexp = + create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); + } + } else { + // Dense layers (first k layers) - GLM uses separate gate/up projections + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0); + } + } + } + break; case LLM_ARCH_BITNET: { model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); From 92b3a36964a23977038c6a561b90fb5ed46222b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:16:45 +0100 Subject: [PATCH 08/55] Update llama.cpp --- src/llama.cpp | 166 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 3e6e76911b..6c47d27f25 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16149,6 +16149,172 @@ struct llm_build_context { return gf; } + struct ggml_cgraph * build_glm4_moe() { + // create a new graph + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + // input embeddings + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); + + // position embeddings + struct ggml_tensor * inp_pos = build_inp_pos(); + + // attention KV cache input + auto * inp_attn = build_attn_inp_kv_unified(); + + // output token IDs (for last layer cropping) + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // Pre-attention norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // Q, K, V projections + struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + } + cb(Qcur, "Qcur", il); + + struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + } + cb(Kcur, "Kcur", il); + + struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + } + cb(Vcur, "Vcur", il); + + // reshape for multi-head + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + // optional Q/K norm (GLM-4.5 variant) + if (model.layers[il].attn_q_norm) { + Qcur = llm_build_norm(ctx0, Qcur, hparams, + model.layers[il].attn_q_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(Qcur, "Qcur_normed", il); + } + if (model.layers[il].attn_k_norm) { + Kcur = llm_build_norm(ctx0, Kcur, hparams, + model.layers[il].attn_k_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(Kcur, "Kcur_normed", il); + } + + // apply RoPE + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + // build attention KV + cur = llm_build_kv(ctx0, lctx, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, /*mask*/ nullptr, + n_tokens, kv_head, n_kv, + 1.0f/sqrtf(float(n_embd_head)), cb, il); + } + + // crop output on last layer + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + // residual connection for attention output + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // FFN / MoE + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + if (static_cast(il) < hparams.n_layer_dense_lead) { + // dense FFN + cur = llm_build_ffn(ctx0, lctx, cur, + model.layers[il].ffn_up, /*gate*/ nullptr, + model.layers[il].ffn_gate, /*unused*/ nullptr, + model.layers[il].ffn_down, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + } else { + // MoE FFN + struct ggml_tensor * moe_out = llm_build_moe_ffn(ctx0, lctx, cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + /*shared_expert=*/nullptr, + hparams.n_expert, hparams.n_expert_used, + LLM_FFN_SILU, /*parallel=*/true, + /*use_group=*/false, 0.0, + LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, + cb, il); + cb(moe_out, "ffn_moe_out", il); + + struct ggml_tensor * shexp_out = llm_build_ffn(ctx0, lctx, cur, + model.layers[il].ffn_up_shexp, + /*gate*/nullptr, + model.layers[il].ffn_gate_shexp, + model.layers[il].ffn_down_shexp, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(shexp_out, "ffn_shexp_out", il); + + cur = ggml_add(ctx0, moe_out, shexp_out); + cb(cur, "ffn_out", il); + } + + // residual and context vector + cur = ggml_add(ctx0, cur, ffn_inp); + cur = llm_build_cvec(ctx0, cur, il); + cb(cur, "l_out", il); + + // prepare next layer input + inpL = cur; + } + + // final norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm head + cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + return gf; + } + struct ggml_cgraph * build_bitnet() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false); From 3012914cbfeabc4340ea5f25eb1198a6f9aaba54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:18:32 +0100 Subject: [PATCH 09/55] Update llama.cpp --- src/llama.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 6c47d27f25..535f22b57f 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -17992,6 +17992,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_glm4(); } break; + case LLM_ARCH_GLM4_MOE: + { + llm = llm.build_glm4_moe(); + } break; case LLM_ARCH_BITNET: { result = llm.build_bitnet(); @@ -21796,6 +21800,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_BERT: case LLM_ARCH_NOMIC_BERT: case LLM_ARCH_STABLELM: + case LLM_ARCH_GLM4_MOE: case LLM_ARCH_BITNET: case LLM_ARCH_BITNET_25: case LLM_ARCH_BITNET_B158: From 3e252ff1e636197b5bc4b756f5708089e9436580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:23:42 +0100 Subject: [PATCH 10/55] Update llama.cpp --- src/llama.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 535f22b57f..f2826c9f1b 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -2656,6 +2656,8 @@ enum e_model { MODEL_70B, MODEL_142B, MODEL_236B, + MODEL_106B_A12B, + MODEL_355B_A32B, MODEL_314B, MODEL_405B, MODEL_671B, From 7a53691b47456cfa81bf4e53ea579f291bab2096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:31:18 +0100 Subject: [PATCH 11/55] Update llama.cpp --- src/llama.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index f2826c9f1b..795556827d 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6105,8 +6105,8 @@ static void llm_load_hparams( // Expert gating function (GLM4_MOE uses sigmoid) ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); - if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { - hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + if (hparams.expert_gating_func == 0) { + hparams.expert_gating_func = LLM_EXPERT_GATING_FUNC_SIGMOID; } switch (hparams.n_layer) { @@ -9021,14 +9021,16 @@ static bool llm_load_tensors( const int64_t n_expert_used = hparams.n_expert_used; const int64_t n_expert_shared = hparams.n_expert_shared; - tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); + model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output - output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); + { + model.output_norm = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + } // if output is NULL, init from the input tok embed - if (output == NULL) { - output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); + if (model.output == NULL) { + model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } for (int i = 0; i < n_layer; ++i) { From a855c755647cc0c0c7f6276e6f3de38d215ec851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:33:41 +0100 Subject: [PATCH 12/55] Update llama.cpp --- src/llama.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 795556827d..b0ea9f31b6 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9034,7 +9034,10 @@ static bool llm_load_tensors( } for (int i = 0; i < n_layer; ++i) { - auto & layer = layers[i]; + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); From 6882e83dc7c1d7d64f77e7b1d03388793e274c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:42:07 +0100 Subject: [PATCH 13/55] Update llama.cpp --- src/llama.cpp | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index b0ea9f31b6..4f528beefc 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9039,25 +9039,25 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); + layer.attn_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); // GLM-style attention with bias terms - layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); - layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, TENSOR_NOT_REQUIRED); - layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, TENSOR_NOT_REQUIRED); - layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, TENSOR_NOT_REQUIRED); + layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); + layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); + layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); + layer.bq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, TENSOR_NOT_REQUIRED); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); + layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); // K/Q norm tensors (optional for GLM-4.5 355B variant) - layer.attn_q_norm = create_tensor( + layer.attn_q_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED); - layer.attn_k_norm = create_tensor( + layer.attn_k_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED); - layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); + layer.ffn_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); // Check if this layer uses MoE or dense FFN based on n_layer_dense_lead const bool use_moe = @@ -9066,7 +9066,7 @@ static bool llm_load_tensors( if (use_moe) { // MoE layers layer.ffn_gate_inp = - create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); + create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); if (n_expert == 0) { GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers"); @@ -9079,28 +9079,28 @@ static bool llm_load_tensors( // MoE branch const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; - layer.ffn_gate_exps = create_tensor( + layer.ffn_gate_exps = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0); - layer.ffn_down_exps = create_tensor( + layer.ffn_down_exps = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0); - layer.ffn_up_exps = create_tensor( + layer.ffn_up_exps = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0); // Shared expert if (n_expert_shared > 0) { const int64_t n_ff_shexp = n_ff_exp * n_expert_shared; - layer.ffn_gate_shexp = create_tensor( + layer.ffn_gate_shexp = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); - layer.ffn_down_shexp = create_tensor( + layer.ffn_down_shexp = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_shexp, n_embd }, 0); layer.ffn_up_shexp = - create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); + create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); } } else { // Dense layers (first k layers) - GLM uses separate gate/up projections - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0); - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0); + layer.ffn_gate = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0); + layer.ffn_down = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0); + layer.ffn_up = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0); } } } From 32026e80f58a73ae37e92715468b89701eacbc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:44:05 +0100 Subject: [PATCH 14/55] Update llama.cpp --- src/llama.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 4f528beefc..302a6aafd3 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9045,17 +9045,17 @@ static bool llm_load_tensors( layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); - layer.bq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, TENSOR_NOT_REQUIRED); - layer.bk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, TENSOR_NOT_REQUIRED); - layer.bv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, TENSOR_NOT_REQUIRED); + layer.bq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); // K/Q norm tensors (optional for GLM-4.5 355B variant) layer.attn_q_norm = create_tensor(ctx_layer, - tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED); + tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, llama_model_loader::TENSOR_NOT_REQUIRED); layer.attn_k_norm = create_tensor(ctx_layer, - tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED); + tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, llama_model_loader::TENSOR_NOT_REQUIRED); layer.ffn_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); From 57d272da6114ccd1569666c3551c1a06b0b0df74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 19:45:34 +0100 Subject: [PATCH 15/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 302a6aafd3..6b3a5eb940 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16281,7 +16281,7 @@ struct llm_build_context { hparams.n_expert, hparams.n_expert_used, LLM_FFN_SILU, /*parallel=*/true, /*use_group=*/false, 0.0, - LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, + LLM_EXPERT_GATING_FUNC_SIGMOID, cb, il); cb(moe_out, "ffn_moe_out", il); From aacba1154bccb6aaa8d8a77cb9e5c73852811521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:13:37 +0100 Subject: [PATCH 16/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 6b3a5eb940..0ceff36aa9 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16173,7 +16173,7 @@ struct llm_build_context { struct ggml_tensor * inp_pos = build_inp_pos(); // attention KV cache input - auto * inp_attn = build_attn_inp_kv_unified(); + //auto * inp_attn = build_attn_inp_kv_unified(); // output token IDs (for last layer cropping) struct ggml_tensor * inp_out_ids = build_inp_out_ids(); From 59b8b88b4458f87e2bea2cc9f0691118d221e6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:16:13 +0100 Subject: [PATCH 17/55] Update llama.cpp --- src/llama.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 0ceff36aa9..66e3bd9c9e 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16265,10 +16265,11 @@ struct llm_build_context { if (static_cast(il) < hparams.n_layer_dense_lead) { // dense FFN cur = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up, /*gate*/ nullptr, - model.layers[il].ffn_gate, /*unused*/ nullptr, - model.layers[il].ffn_down, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); cb(cur, "ffn_out", il); } else { // MoE FFN From 299c23a9042fc5f0f6914f3f61c376918332e96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:19:23 +0100 Subject: [PATCH 18/55] Update llama.cpp --- src/llama.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 66e3bd9c9e..cfbbeb5fe5 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16287,10 +16287,10 @@ struct llm_build_context { cb(moe_out, "ffn_moe_out", il); struct ggml_tensor * shexp_out = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up_shexp, - /*gate*/nullptr, - model.layers[il].ffn_gate_shexp, - model.layers[il].ffn_down_shexp, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, cb, il); cb(shexp_out, "ffn_shexp_out", il); From 019ff22494cd93144f43ef07c207f0e33b0aa511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:20:43 +0100 Subject: [PATCH 19/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index cfbbeb5fe5..33b4df323d 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16300,7 +16300,7 @@ struct llm_build_context { // residual and context vector cur = ggml_add(ctx0, cur, ffn_inp); - cur = llm_build_cvec(ctx0, cur, il); + cur = lctx.cvec.apply_to(ctx0, cur, il); cb(cur, "l_out", il); // prepare next layer input From fba5973f3d639656dd2a10c2bf3ff770d62572d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:21:57 +0100 Subject: [PATCH 20/55] Update llama.cpp --- src/llama.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 33b4df323d..1a2a7f1924 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16306,6 +16306,8 @@ struct llm_build_context { // prepare next layer input inpL = cur; } + + cur = inpL; // final norm cur = llm_build_norm(ctx0, inpL, hparams, @@ -16317,8 +16319,7 @@ struct llm_build_context { // lm head cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); cb(cur, "result_output", -1); - res->t_logits = cur; - + ggml_build_forward_expand(gf, cur); return gf; } From d28c8a4ab0677099283acb8b1b52b8184c277598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:22:46 +0100 Subject: [PATCH 21/55] Update llama.cpp --- src/llama.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 1a2a7f1924..b0d98ce54c 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16314,7 +16314,6 @@ struct llm_build_context { model.output_norm, NULL, LLM_NORM_RMS, cb, -1); cb(cur, "result_norm", -1); - res->t_embd = cur; // lm head cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); From ff0c36899d23741c4b2b13973cf0fc84ebb6c65c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:23:58 +0100 Subject: [PATCH 22/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index b0d98ce54c..4cf5a37f94 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -18002,7 +18002,7 @@ static struct ggml_cgraph * llama_build_graph( } break; case LLM_ARCH_GLM4_MOE: { - llm = llm.build_glm4_moe(); + result = llm.build_glm4_moe(); } break; case LLM_ARCH_BITNET: { From 6258a120d5d75cf54624f158a63f0d922088978a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:32:13 +0100 Subject: [PATCH 23/55] Update llama.cpp --- src/llama.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 4cf5a37f94..53efbf92bb 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -20455,7 +20455,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // TODO: avoid hardcoded tensor names - use the TN_* constants if (name.find("attn_v.weight") != std::string::npos || - name.find("attn_qkv.weight") != std::string::npos) { + name.find("attn_qkv.weight") != std::string::npos || + name.find("eh_proj.weight") != std::string::npos || + name.find("shared_head.head") != std::string::npos) { ++qs.n_attention_wv; } else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) { qs.has_output = true; From 9b1fdfff3193139c21ef3eea64cf141401eb9615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:36:38 +0100 Subject: [PATCH 24/55] Revert "Update llama.cpp" This reverts commit ff0c36899d23741c4b2b13973cf0fc84ebb6c65c. --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 53efbf92bb..1a4ef54a8e 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -18002,7 +18002,7 @@ static struct ggml_cgraph * llama_build_graph( } break; case LLM_ARCH_GLM4_MOE: { - result = llm.build_glm4_moe(); + llm = llm.build_glm4_moe(); } break; case LLM_ARCH_BITNET: { From 746b87fabd18b1fc806f41cbae347681fd8edb45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:46:29 +0100 Subject: [PATCH 25/55] Update llama.cpp --- src/llama.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 1a4ef54a8e..1debe87fdf 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -20456,11 +20456,14 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // TODO: avoid hardcoded tensor names - use the TN_* constants if (name.find("attn_v.weight") != std::string::npos || name.find("attn_qkv.weight") != std::string::npos || - name.find("eh_proj.weight") != std::string::npos || + name.find("eh_proj") != std::string::npos || name.find("shared_head.head") != std::string::npos) { ++qs.n_attention_wv; } else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) { qs.has_output = true; + } else { + // for debugging, see which names aren't matching + fprintf(stderr, "[quant] skipping name = %s\n", name.c_str()); } } From 9ce87a2fb573223d906903f1c846e1e68822f8ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:47:19 +0100 Subject: [PATCH 26/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 1debe87fdf..dd8c20feb3 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -18002,7 +18002,7 @@ static struct ggml_cgraph * llama_build_graph( } break; case LLM_ARCH_GLM4_MOE: { - llm = llm.build_glm4_moe(); + result = llm.build_glm4_moe(); } break; case LLM_ARCH_BITNET: { From 92f5b07899d46aa8fabe3ac16b84e84014260c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 20:54:56 +0100 Subject: [PATCH 27/55] Update llama.cpp --- src/llama.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index dd8c20feb3..e453983953 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -20455,15 +20455,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // TODO: avoid hardcoded tensor names - use the TN_* constants if (name.find("attn_v.weight") != std::string::npos || - name.find("attn_qkv.weight") != std::string::npos || - name.find("eh_proj") != std::string::npos || - name.find("shared_head.head") != std::string::npos) { + name.find("attn_qkv.weight") != std::string::npos) { ++qs.n_attention_wv; } else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) { qs.has_output = true; - } else { - // for debugging, see which names aren't matching - fprintf(stderr, "[quant] skipping name = %s\n", name.c_str()); } } @@ -20488,8 +20483,18 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // - qs.n_attention_wv == 3 * model.hparams.n_layer for Encoder-Decoder models // - model.arch == LLM_ARCH_DECI for Deci-Nemotron models // - GGML_ASSERT((qs.n_attention_wv == 0 || qs.n_attention_wv == (int)model.hparams.n_layer || qs.n_attention_wv == 3 * (int)model.hparams.n_layer || model.arch == LLM_ARCH_DECI) && "n_attention_wv is unexpected"); - + //GGML_ASSERT((qs.n_attention_wv == 0 || qs.n_attention_wv == (int)model.hparams.n_layer || qs.n_attention_wv == 3 * (int)model.hparams.n_layer || model.arch == LLM_ARCH_DECI) && "n_attention_wv is unexpected"); + // allow any count for GLM4-MoE, but still enforce for all others + if (model.arch != LLM_ARCH_GLM4_MOE) { + GGML_ASSERT( + qs.n_attention_wv == 0 + || qs.n_attention_wv == (int)model.hparams.n_layer + || qs.n_attention_wv == 3 * (int)model.hparams.n_layer + || model.arch == LLM_ARCH_DECI + && "n_attention_wv is unexpected" + ); + } + size_t total_size_org = 0; size_t total_size_new = 0; From 500b27365a56332e177b2007c4a1d8a8d04546b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 21:53:34 +0100 Subject: [PATCH 28/55] Update llama.cpp --- src/llama.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index e453983953..a76629a0c5 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6094,6 +6094,9 @@ static void llm_load_hparams( } break; case LLM_ARCH_GLM4_MOE: { + // bump n_layer to account for the extra NextN block + hparams.n_layer += 1; + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); From 18f9c5bb17595fcd2d1ebfe3159d60c297ff2b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 22:28:13 +0100 Subject: [PATCH 29/55] Revert "Update llama.cpp" This reverts commit 500b27365a56332e177b2007c4a1d8a8d04546b3. --- src/llama.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index a76629a0c5..e453983953 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6094,9 +6094,6 @@ static void llm_load_hparams( } break; case LLM_ARCH_GLM4_MOE: { - // bump n_layer to account for the extra NextN block - hparams.n_layer += 1; - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); From 4cabc501e7a680da9b39b65f3aaca79dee47a8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Tue, 29 Jul 2025 22:29:49 +0100 Subject: [PATCH 30/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 40a99c0036..edcdf6be04 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -3952,6 +3952,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): class Glm4MoeModel(TextModel): model_arch = gguf.MODEL_ARCH.GLM4_MOE + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # GLM4_MOE has num_hidden_layers + 1 actual layers (including NextN layer) + self.block_count = self.hparams["num_hidden_layers"] + 1 + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + def set_vocab(self): from transformers import AutoTokenizer From e06c5dbf086753b2f0c0b417d7fd2df9eb9c3e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 00:11:43 +0100 Subject: [PATCH 31/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index edcdf6be04..13ccd34918 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -4028,8 +4028,8 @@ def modify_tensors( elif name.startswith("model.language_model."): name = name.replace("language_model.", "") # for multimodal variants - # Handle main token embedding - if name == "model.embed_tokens.weight": + # Handle main token embedding (but not layer-specific NextN embeddings) + if name == "model.embed_tokens.weight" and ".layers." not in name: return [(self.map_tensor_name("token_embd.weight"), data_torch)] # Handle routed experts From 36431e7adb2e95cf0a8a9833564191af81a965be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 07:21:59 +0100 Subject: [PATCH 32/55] Update llama.cpp --- src/llama.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llama.cpp b/src/llama.cpp index e453983953..b73274e33d 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9060,6 +9060,7 @@ static bool llm_load_tensors( layer.ffn_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); // Check if this layer uses MoE or dense FFN based on n_layer_dense_lead + // GLM 4.5 uses hybrid architecture: layer 0 is dense, layers 1+ are MoE const bool use_moe = (hparams.n_expert > 0) && (static_cast(i) >= hparams.n_layer_dense_lead); From b173be31ddb273d50c51f44eae0fb548a11ad8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 11:03:34 +0100 Subject: [PATCH 33/55] Update llama.cpp --- src/llama.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index b73274e33d..93b740084f 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9032,6 +9032,40 @@ static bool llm_load_tensors( if (model.output == NULL) { model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } + + // --- NextN / MTP tensors (preserved but unused), on the final layer --- + { + const int final_layer = n_layer - 1; + // EH_PROJ: [2*embd, embd] + create_tensor(ctx_for_layer(final_layer), + tn(LLM_TENSOR_NEXTN_EH_PROJ, final_layer), + { 2*n_embd, n_embd }, + llama_model_loader::TENSOR_NOT_REQUIRED); + // EMBED_TOKENS: [embd, vocab] + create_tensor(ctx_for_layer(final_layer), + tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, final_layer), + { n_embd, n_vocab }, + llama_model_loader::TENSOR_NOT_REQUIRED); + // ENORM, HNORM: [embd] + create_tensor(ctx_for_layer(final_layer), + tn(LLM_TENSOR_NEXTN_ENORM, final_layer), + { n_embd }, + llama_model_loader::TENSOR_NOT_REQUIRED); + create_tensor(ctx_for_layer(final_layer), + tn(LLM_TENSOR_NEXTN_HNORM, final_layer), + { n_embd }, + llama_model_loader::TENSOR_NOT_REQUIRED); + // SHARED_HEAD_HEAD: [embd, vocab] + create_tensor(ctx_for_layer(final_layer), + tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, final_layer), + { n_embd, n_vocab }, + llama_model_loader::TENSOR_NOT_REQUIRED); + // SHARED_HEAD_NORM: [embd] + create_tensor(ctx_for_layer(final_layer), + tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, final_layer), + { n_embd }, + llama_model_loader::TENSOR_NOT_REQUIRED); + } for (int i = 0; i < n_layer; ++i) { ggml_context * ctx_layer = ctx_for_layer(i); @@ -9068,6 +9102,10 @@ static bool llm_load_tensors( // MoE layers layer.ffn_gate_inp = create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); + // gate bias + layer.ffn_exp_probs_b = + create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), { n_expert }, + llama_model_loader::TENSOR_NOT_REQUIRED); if (n_expert == 0) { GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers"); From 19bb8859fd72faf9da2f2fa0415cca5a2b5cbd0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 11:39:49 +0100 Subject: [PATCH 34/55] Testing different MoE FFN --- src/llama.cpp | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 93b740084f..87110348f7 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16277,7 +16277,7 @@ struct llm_build_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - // build attention KV + // build attention KV (no unified cache) cur = llm_build_kv(ctx0, lctx, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, /*mask*/ nullptr, @@ -16312,17 +16312,18 @@ struct llm_build_context { cb(cur, "ffn_out", il); } else { // MoE FFN - struct ggml_tensor * moe_out = llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - /*shared_expert=*/nullptr, - hparams.n_expert, hparams.n_expert_used, - LLM_FFN_SILU, /*parallel=*/true, - /*use_group=*/false, 0.0, - LLM_EXPERT_GATING_FUNC_SIGMOID, - cb, il); + ggml_tensor * moe_out = + llm_build_moe_ffn(ctx0, lctx, cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + true, hparams.expert_weights_scale, + (enum llm_expert_gating_func_type) hparams.expert_gating_func, + cb, il); cb(moe_out, "ffn_moe_out", il); struct ggml_tensor * shexp_out = llm_build_ffn(ctx0, lctx, cur, From e941116cc6ab4cc20b9a77dc0c9e36c02c38af4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 16:57:33 +0100 Subject: [PATCH 35/55] Update llama.cpp --- src/llama.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 87110348f7..6f5599966a 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16312,18 +16312,17 @@ struct llm_build_context { cb(cur, "ffn_out", il); } else { // MoE FFN - ggml_tensor * moe_out = - llm_build_moe_ffn(ctx0, lctx, cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - model.layers[il].ffn_exp_probs_b, - n_expert, n_expert_used, - LLM_FFN_SILU, hparams.expert_weights_norm, - true, hparams.expert_weights_scale, - (enum llm_expert_gating_func_type) hparams.expert_gating_func, - cb, il); + struct ggml_tensor * moe_out = llm_build_moe_ffn(ctx0, lctx, cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + /*shared_expert=*/nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, /*parallel=*/true, + /*use_group=*/false, 0.0, + (enum llm_expert_gating_func_type) hparams.expert_gating_func, + cb, il); cb(moe_out, "ffn_moe_out", il); struct ggml_tensor * shexp_out = llm_build_ffn(ctx0, lctx, cur, From f5a5a0d0e3a836b5287e54b7fc6987f33d98c0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 17:28:33 +0100 Subject: [PATCH 36/55] Update llama.cpp --- src/llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 6f5599966a..08d62832c1 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16317,10 +16317,10 @@ struct llm_build_context { model.layers[il].ffn_up_exps, model.layers[il].ffn_gate_exps, model.layers[il].ffn_down_exps, - /*shared_expert=*/nullptr, + model.layers[il].ffn_exp_probs_b, n_expert, n_expert_used, LLM_FFN_SILU, /*parallel=*/true, - /*use_group=*/false, 0.0, + /*use_group=*/false, hparams.norm_topk_prob ? 1.0f : 0.0f, (enum llm_expert_gating_func_type) hparams.expert_gating_func, cb, il); cb(moe_out, "ffn_moe_out", il); From 6f4eec89ba50e2c607e729c164dd0e6b0fdd05cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 19:17:51 +0100 Subject: [PATCH 37/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 08d62832c1..c12ac26f70 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16320,7 +16320,7 @@ struct llm_build_context { model.layers[il].ffn_exp_probs_b, n_expert, n_expert_used, LLM_FFN_SILU, /*parallel=*/true, - /*use_group=*/false, hparams.norm_topk_prob ? 1.0f : 0.0f, + /*use_group=*/false, /*renormalize=*/1.0f, // always normalize top‐k (enum llm_expert_gating_func_type) hparams.expert_gating_func, cb, il); cb(moe_out, "ffn_moe_out", il); From a9531982386f9dc23fa608a0c5e1572ff3df78ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Wed, 30 Jul 2025 21:15:07 +0100 Subject: [PATCH 38/55] Update llama.cpp --- src/llama.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index c12ac26f70..bdbc3da104 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6465,7 +6465,7 @@ static void llm_load_vocab( vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO; vocab.tokenizer_clean_spaces = false; } else if ( - tokenizer_pre == "glm4" || + tokenizer_pre == "glm4" || tokenizer_pre == "glm4moe" || tokenizer_pre == "chatglm-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHATGLM4; vocab.special_bos_id = -1; @@ -10403,7 +10403,7 @@ static struct ggml_tensor * llm_build_kqv( // For DeepSeek-2, it is perfectly fine with fp16 for PP, but I get gibberish when uding fp16 for TG. // Not sure if it is really a matter of insufficient precision, or I have made a mistake in the fattn-vec-f16 kernel. if (use_f32_precision || model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || - (model.arch == LLM_ARCH_DEEPSEEK2 && q->ne[1] <= 8) || model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_GLM4) { + (model.arch == LLM_ARCH_DEEPSEEK2 && q->ne[1] <= 8) || model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) { ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); } //ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); @@ -10428,7 +10428,7 @@ static struct ggml_tensor * llm_build_kqv( //ggml_mul_mat_set_prec(kq, GGML_PREC_F32); if (use_f32_precision || model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || - model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_GLM4) { + model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) { // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847 ggml_mul_mat_set_prec(kq, GGML_PREC_F32); From 660b0b95e447efd2e81856e760a446d094852a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 03:37:41 +0100 Subject: [PATCH 39/55] Update llama.cpp --- src/llama.cpp | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index bdbc3da104..28d802e369 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16286,7 +16286,9 @@ struct llm_build_context { } // crop output on last layer - if (il == n_layer - 1 && inp_out_ids) { + if (il == n_layer - 1) { + // skip computing output for unused tokens + ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -16301,7 +16303,7 @@ struct llm_build_context { LLM_NORM_RMS, cb, il); cb(cur, "ffn_norm", il); - if (static_cast(il) < hparams.n_layer_dense_lead) { + if ((uint32_t) il < hparams.n_layer_dense_lead) { // dense FFN cur = llm_build_ffn(ctx0, lctx, cur, model.layers[il].ffn_up, NULL, NULL, @@ -16319,22 +16321,24 @@ struct llm_build_context { model.layers[il].ffn_down_exps, model.layers[il].ffn_exp_probs_b, n_expert, n_expert_used, - LLM_FFN_SILU, /*parallel=*/true, - /*use_group=*/false, /*renormalize=*/1.0f, // always normalize top‐k + LLM_FFN_SILU, hparams.expert_weights_norm, + true, hparams.expert_weights_scale, (enum llm_expert_gating_func_type) hparams.expert_gating_func, cb, il); cb(moe_out, "ffn_moe_out", il); - - struct ggml_tensor * shexp_out = llm_build_ffn(ctx0, lctx, cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, - nullptr, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(shexp_out, "ffn_shexp_out", il); - - cur = ggml_add(ctx0, moe_out, shexp_out); - cb(cur, "ffn_out", il); + + { + struct ggml_tensor * shexp_out = llm_build_ffn(ctx0, lctx, cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(shexp_out, "ffn_shexp_out", il); + + cur = ggml_add(ctx0, moe_out, shexp_out); + cb(cur, "ffn_out", il); + } } // residual and context vector @@ -16349,7 +16353,7 @@ struct llm_build_context { cur = inpL; // final norm - cur = llm_build_norm(ctx0, inpL, hparams, + cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, NULL, LLM_NORM_RMS, cb, -1); cb(cur, "result_norm", -1); From 7e60282d65c06800c71b3fbb34623612fca4fcd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 03:58:36 +0100 Subject: [PATCH 40/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 13ccd34918..c5cd3ce911 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -618,6 +618,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b": # ref: https://huggingface.co/THUDM/glm-4-9b-chat res = "chatglm-bpe" + if chkhsh == "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902": + # ref: https://huggingface.co/zai-org/GLM-4.5-Air, https://huggingface.co/zai-org/GLM-4.5 + res = "gpt-2" if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee": # ref: https://huggingface.co/LumiOpen/Viking-7B res = "viking" From a1a7b3e2e50627efa3dfe9a247032a37e559c690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:05:34 +0100 Subject: [PATCH 41/55] Update llama.cpp --- src/llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 28d802e369..8a93b32339 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6465,7 +6465,7 @@ static void llm_load_vocab( vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO; vocab.tokenizer_clean_spaces = false; } else if ( - tokenizer_pre == "glm4" || tokenizer_pre == "glm4moe" || + tokenizer_pre == "glm4" || tokenizer_pre == "chatglm-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHATGLM4; vocab.special_bos_id = -1; @@ -10488,7 +10488,7 @@ static struct ggml_tensor * llm_build_kqv( auto q_i = ggml_view_3d(ctx, q, q->ne[0], q->ne[1], this_ne12, q->nb[1], q->nb[2], q->nb[2]*i12); auto kq_i = ggml_mul_mat(ctx, k_i, q_i); if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || - model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_GLM4) { + model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) { ggml_mul_mat_set_prec(kq_i, GGML_PREC_F32); } if (model.arch == LLM_ARCH_GROK) { From 97d87c7674744eb7753c5f825ff6ed093d448545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:09:00 +0100 Subject: [PATCH 42/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index c5cd3ce911..afd59b07f1 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -4311,6 +4311,21 @@ def set_vocab(self): special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # this one is usually not in config.json anyway special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338 + + # Fix chat template syntax error in GLM-4.5 models + if special_vocab.chat_template and isinstance(special_vocab.chat_template, str): + # Fix multiple syntax issues in GLM-4.5 chat template + template = special_vocab.chat_template + # Fix nested double quotes issue + template = template.replace('endswith("/nothink")', "endswith('/nothink')") + # Fix any other potential parentheses/tuple issues + template = template.replace( + "not visible_text(m.content).endswith('/nothink'))", + "not visible_text(m.content).endswith('/nothink')" + ) + special_vocab.chat_template = template + special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): From 807cdcb7151af26c298482e7d2debb3c86b49943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:23:23 +0100 Subject: [PATCH 43/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index afd59b07f1..c274882207 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -3973,19 +3973,32 @@ def set_vocab(self): self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) - special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + + # Set special tokens special_vocab._set_special_token( "eos", tokenizer.get_added_vocab()["<|endoftext|>"] ) special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) - special_vocab._set_special_token("eog", tokenizer.get_added_vocab()["<|user|>"]) - special_vocab._set_special_token("eog", tokenizer.get_added_vocab()["<|observation|>"]) special_vocab._set_special_token( "unk", tokenizer.get_added_vocab()["<|endoftext|>"] ) special_vocab._set_special_token( "bos", tokenizer.get_added_vocab()["<|endoftext|>"] ) + special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338 + + # Fix chat template syntax error in GLM-4.5 models + if special_vocab.chat_template and isinstance(special_vocab.chat_template, str): + # Fix multiple syntax issues in GLM-4.5 chat template + template = special_vocab.chat_template + # Fix nested double quotes issue + template = template.replace('endswith("/nothink")', "endswith('/nothink')") + # Fix any other potential parentheses/tuple issues + template = template.replace( + "not visible_text(m.content).endswith('/nothink'))", + "not visible_text(m.content).endswith('/nothink')" + ) + special_vocab.chat_template = template special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): @@ -4311,21 +4324,6 @@ def set_vocab(self): special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # this one is usually not in config.json anyway special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) - special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338 - - # Fix chat template syntax error in GLM-4.5 models - if special_vocab.chat_template and isinstance(special_vocab.chat_template, str): - # Fix multiple syntax issues in GLM-4.5 chat template - template = special_vocab.chat_template - # Fix nested double quotes issue - template = template.replace('endswith("/nothink")', "endswith('/nothink')") - # Fix any other potential parentheses/tuple issues - template = template.replace( - "not visible_text(m.content).endswith('/nothink'))", - "not visible_text(m.content).endswith('/nothink')" - ) - special_vocab.chat_template = template - special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): From 697960713ab0753318b9df940cefeea1215d7df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:26:03 +0100 Subject: [PATCH 44/55] Update llama.cpp --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 8a93b32339..47c0cd7f78 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -1439,7 +1439,7 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" }, { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" }, { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" }, - // NextN/MTP tensors - preserved but unused + // NextN/MTP tensors - preserved but unused (in final layer, dynamic layer number) { LLM_TENSOR_NEXTN_EH_PROJ, "blk.%d.eh_proj" }, { LLM_TENSOR_NEXTN_EMBED_TOKENS, "blk.%d.embed_tokens" }, { LLM_TENSOR_NEXTN_ENORM, "blk.%d.enorm" }, From ef92e5cfa02d9d84b2539e4e72ea9b9300f4ff4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:30:43 +0100 Subject: [PATCH 45/55] Update llama.cpp --- src/llama.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 47c0cd7f78..b395ed8f29 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6102,6 +6102,8 @@ static void llm_load_hparams( ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, 0); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, 0); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, 0); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); // Expert gating function (GLM4_MOE uses sigmoid) ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); From 74dadf3a89d0e62c45a8772c5b3b3cb809a3e026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:41:41 +0100 Subject: [PATCH 46/55] Update llama.cpp --- src/llama.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index b395ed8f29..16afe86cc3 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9023,12 +9023,12 @@ static bool llm_load_tensors( const int64_t n_expert_used = hparams.n_expert_used; const int64_t n_expert_shared = hparams.n_expert_shared; - model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // output { - model.output_norm = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + model.output_norm = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + model.output = create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); } // if output is NULL, init from the input tok embed if (model.output == NULL) { @@ -9081,9 +9081,9 @@ static bool llm_load_tensors( layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); - layer.bq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.bk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.bv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bq = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); From 83d2bb3e2d8a0a630f77b225515a52c48d4fe16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 04:57:48 +0100 Subject: [PATCH 47/55] Update llama.cpp - KQ_mask --- src/llama.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 16afe86cc3..62a57080f5 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16215,6 +16215,8 @@ struct llm_build_context { // attention KV cache input //auto * inp_attn = build_attn_inp_kv_unified(); + + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); // output token IDs (for last layer cropping) struct ggml_tensor * inp_out_ids = build_inp_out_ids(); @@ -16254,7 +16256,7 @@ struct llm_build_context { Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - // optional Q/K norm (GLM-4.5 variant) + // Apply Q/K norm if available (GLM-4.5 355B variant) if (model.layers[il].attn_q_norm) { Qcur = llm_build_norm(ctx0, Qcur, hparams, model.layers[il].attn_q_norm, NULL, @@ -16282,7 +16284,7 @@ struct llm_build_context { // build attention KV (no unified cache) cur = llm_build_kv(ctx0, lctx, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, /*mask*/ nullptr, + Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } From 4250fb4e91364efdccea09947d8c8f9f43a6ecde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Thu, 31 Jul 2025 08:59:57 +0100 Subject: [PATCH 48/55] Update llama.cpp --- src/llama.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 62a57080f5..88d8456090 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -1439,6 +1439,7 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" }, { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" }, { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" }, + { LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" }, // NextN/MTP tensors - preserved but unused (in final layer, dynamic layer number) { LLM_TENSOR_NEXTN_EH_PROJ, "blk.%d.eh_proj" }, { LLM_TENSOR_NEXTN_EMBED_TOKENS, "blk.%d.embed_tokens" }, @@ -9035,9 +9036,9 @@ static bool llm_load_tensors( model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } - // --- NextN / MTP tensors (preserved but unused), on the final layer --- + // NextN/MTP tensors (preserved but unused) - only in final layer (46 for Air, 92 for GLM-4.5) { - const int final_layer = n_layer - 1; + const int final_layer = n_layer - 1; // NextN tensors are in the last layer only // EH_PROJ: [2*embd, embd] create_tensor(ctx_for_layer(final_layer), tn(LLM_TENSOR_NEXTN_EH_PROJ, final_layer), @@ -9081,9 +9082,9 @@ static bool llm_load_tensors( layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); - layer.bq = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.bk = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.bv = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bq = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, 0); + layer.bk = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, 0); + layer.bv = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, 0); layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); @@ -9106,8 +9107,8 @@ static bool llm_load_tensors( create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); // gate bias layer.ffn_exp_probs_b = - create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), { n_expert }, - llama_model_loader::TENSOR_NOT_REQUIRED); + create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_EXP_PROBS_B, i), { n_expert }, + 0); if (n_expert == 0) { GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers"); From 0040a42ef5cdb16c4974b83b21f9eb47ec7b5223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Fri, 1 Aug 2025 08:26:48 +0100 Subject: [PATCH 49/55] Revert "Update llama.cpp" This reverts commit 4250fb4e91364efdccea09947d8c8f9f43a6ecde. --- src/llama.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 88d8456090..62a57080f5 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -1439,7 +1439,6 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" }, { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" }, { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" }, - { LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" }, // NextN/MTP tensors - preserved but unused (in final layer, dynamic layer number) { LLM_TENSOR_NEXTN_EH_PROJ, "blk.%d.eh_proj" }, { LLM_TENSOR_NEXTN_EMBED_TOKENS, "blk.%d.embed_tokens" }, @@ -9036,9 +9035,9 @@ static bool llm_load_tensors( model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } - // NextN/MTP tensors (preserved but unused) - only in final layer (46 for Air, 92 for GLM-4.5) + // --- NextN / MTP tensors (preserved but unused), on the final layer --- { - const int final_layer = n_layer - 1; // NextN tensors are in the last layer only + const int final_layer = n_layer - 1; // EH_PROJ: [2*embd, embd] create_tensor(ctx_for_layer(final_layer), tn(LLM_TENSOR_NEXTN_EH_PROJ, final_layer), @@ -9082,9 +9081,9 @@ static bool llm_load_tensors( layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); - layer.bq = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, 0); - layer.bk = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, 0); - layer.bv = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, 0); + layer.bq = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); @@ -9107,8 +9106,8 @@ static bool llm_load_tensors( create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); // gate bias layer.ffn_exp_probs_b = - create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_EXP_PROBS_B, i), { n_expert }, - 0); + create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), { n_expert }, + llama_model_loader::TENSOR_NOT_REQUIRED); if (n_expert == 0) { GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers"); From 9640fe9a98968d170e1fc032e9fe6af408af9ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Fri, 1 Aug 2025 20:41:42 +0100 Subject: [PATCH 50/55] Update llama.cpp - Fix non-fa ppl Suggested by @ubergarm - https://github.com/ikawrakow/ik_llama.cpp/pull/662#issuecomment-3145535053 --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index 62a57080f5..5fe20a8fe8 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16254,7 +16254,7 @@ struct llm_build_context { // reshape for multi-head Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + // Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply Q/K norm if available (GLM-4.5 355B variant) if (model.layers[il].attn_q_norm) { From ba74150f01159b2a35017cf6f1bb0dea633a9b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sat, 2 Aug 2025 06:40:16 +0100 Subject: [PATCH 51/55] Update convert_hf_to_gguf.py - ik_llama bugfix From @ubergarm - https://github.com/ikawrakow/ik_llama.cpp/pull/668#issuecomment-3145913701 --- convert_hf_to_gguf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index c274882207..3e46850727 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -618,6 +618,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b": # ref: https://huggingface.co/THUDM/glm-4-9b-chat res = "chatglm-bpe" + if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2": + # ref: https://huggingface.co/THUDM/glm-4-9b-hf + res = "glm4" if chkhsh == "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902": # ref: https://huggingface.co/zai-org/GLM-4.5-Air, https://huggingface.co/zai-org/GLM-4.5 res = "gpt-2" @@ -3951,8 +3954,8 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): return [(self.map_tensor_name(name), data_torch)] return super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Glm4MoeForCausalLM") -class Glm4MoeModel(TextModel): +@Model.register("Glm4MoeForCausalLM") +class Glm4MoeModel(Model): model_arch = gguf.MODEL_ARCH.GLM4_MOE def __init__(self, *args, **kwargs): From a6c22f01af673fb1122f135f3ab557988f5e733a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sat, 2 Aug 2025 06:54:26 +0100 Subject: [PATCH 52/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 3e46850727..8d7855cfa9 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -4096,7 +4096,7 @@ def modify_tensors( # Handle expert gating input (routing gate) if ".mlp.gate.e_score_correction_bias" in name: new_name = name.replace("model.layers.", "blk.").replace( - ".mlp.gate.e_score_correction_bias", ".ffn_gate_inp.bias" + ".mlp.gate.e_score_correction_bias", ".ffn_gate_inp.bias" # *NOTE* this is ".exp_probs_b" in mainline PR ) return [(new_name, data_torch)] elif ".mlp.gate.weight" in name: @@ -4106,6 +4106,7 @@ def modify_tensors( return [(new_name, data_torch)] # Handle shared expert tensors + # What are these MTP tensors? If we preserve them do they need to be "noop" or whatever? if ".mlp.shared_experts." in name: new_name = name.replace("model.layers.", "blk.").replace(".mlp.shared_experts.", ".ffn_") if "gate_proj" in new_name: @@ -4134,7 +4135,7 @@ def modify_tensors( new_name = name return [(self.map_tensor_name(new_name), data_torch)] - # Handle special NextN tensors - preserve for future MTP support + # Handle special NextN tensors - preserve for future MTP support - See https://github.com/ggml-org/llama.cpp/pull/13236 if ( ".embed_tokens." in name or ".shared_head." in name @@ -4143,6 +4144,7 @@ def modify_tensors( or ".hnorm." in name ): new_name = name.replace("model.layers.", "blk.").replace("model.", "").replace(".weight", "") + logger.debug(f"Skipping MTP tensor: {new_name}") return [(new_name, data_torch)] # GLM tensor mapping - handle directly without map_tensor_name From 08f30c9efd8e82167b5a774dc08b4e45a9ac944f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sat, 2 Aug 2025 06:56:14 +0100 Subject: [PATCH 53/55] Update convert_hf_to_gguf.py --- convert_hf_to_gguf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 8d7855cfa9..be2eb17021 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -4106,7 +4106,6 @@ def modify_tensors( return [(new_name, data_torch)] # Handle shared expert tensors - # What are these MTP tensors? If we preserve them do they need to be "noop" or whatever? if ".mlp.shared_experts." in name: new_name = name.replace("model.layers.", "blk.").replace(".mlp.shared_experts.", ".ffn_") if "gate_proj" in new_name: @@ -4144,7 +4143,7 @@ def modify_tensors( or ".hnorm." in name ): new_name = name.replace("model.layers.", "blk.").replace("model.", "").replace(".weight", "") - logger.debug(f"Skipping MTP tensor: {new_name}") + # logger.debug(f"Skipping MTP tensor: {new_name}") return [(new_name, data_torch)] # GLM tensor mapping - handle directly without map_tensor_name From 4e10857abe9a1dbf18edcb3b4f47b2c6bfcd4db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sun, 3 Aug 2025 08:44:58 +0100 Subject: [PATCH 54/55] Revert to proper version that produced good BF16 https://github.com/ikawrakow/ik_llama.cpp/pull/668#issuecomment-3147374559 --- convert_hf_to_gguf.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index be2eb17021..464973cc8b 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -3988,20 +3988,7 @@ def set_vocab(self): special_vocab._set_special_token( "bos", tokenizer.get_added_vocab()["<|endoftext|>"] ) - special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338 - - # Fix chat template syntax error in GLM-4.5 models - if special_vocab.chat_template and isinstance(special_vocab.chat_template, str): - # Fix multiple syntax issues in GLM-4.5 chat template - template = special_vocab.chat_template - # Fix nested double quotes issue - template = template.replace('endswith("/nothink")', "endswith('/nothink')") - # Fix any other potential parentheses/tuple issues - template = template.replace( - "not visible_text(m.content).endswith('/nothink'))", - "not visible_text(m.content).endswith('/nothink')" - ) - special_vocab.chat_template = template + special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): @@ -4048,7 +4035,7 @@ def modify_tensors( name = name.replace("language_model.", "") # for multimodal variants # Handle main token embedding (but not layer-specific NextN embeddings) - if name == "model.embed_tokens.weight" and ".layers." not in name: + if name == "model.embed_tokens.weight": return [(self.map_tensor_name("token_embd.weight"), data_torch)] # Handle routed experts From fe552b9e6129920c32cc782e28655df5141667af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sun, 3 Aug 2025 08:45:30 +0100 Subject: [PATCH 55/55] Support for jinja chat_template files --- gguf-py/gguf/vocab.py | 395 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 388 insertions(+), 7 deletions(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index cca0979862..e1d5aaf47a 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -1,5 +1,6 @@ from __future__ import annotations +from enum import Enum import re import logging import json @@ -7,7 +8,29 @@ from pathlib import Path from typing import Any, Callable, Sequence, Mapping, Iterable, Protocol, ClassVar, runtime_checkable -from sentencepiece import SentencePieceProcessor +try: + from sentencepiece import SentencePieceProcessor +except ImportError: + SentencePieceProcessor = None + +try: + from mistral_common.tokens.tokenizers.mistral import MistralTokenizer + from mistral_common.tokens.tokenizers.tekken import Tekkenizer + from mistral_common.tokens.tokenizers.utils import ( + _filter_valid_tokenizer_files, + ) + from mistral_common.tokens.tokenizers.sentencepiece import ( + SentencePieceTokenizer, + ) +except ImportError: + _mistral_common_installed = False + MistralTokenizer = None + Tekkenizer = None + SentencePieceTokenizer = None + _filter_valid_tokenizer_files = None +else: + _mistral_common_installed = True + import gguf @@ -116,6 +139,7 @@ def _set_special_token(self, typ: str, tid: Any) -> None: logger.warning(f'Special token type {typ}, id {tid} out of range, must be under {self.n_vocab} - skipping') def _try_load_from_tokenizer_json(self, path: Path) -> bool: + tokenizer = None tokenizer_file = path / 'tokenizer.json' if tokenizer_file.is_file(): with open(tokenizer_file, encoding = 'utf-8') as f: @@ -149,15 +173,110 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: added_tokens = tokenizer.get('added_tokens', {}) else: added_tokens = {} + tokenizer_config = None tokenizer_config_file = path / 'tokenizer_config.json' - if not tokenizer_config_file.is_file(): + if tokenizer_config_file.is_file(): + with open(tokenizer_config_file, encoding = 'utf-8') as f: + tokenizer_config = json.load(f) + if tokenizer: + special_bos = (tokenizer_config or {}).get('bos_token') + special_cls = (tokenizer_config or {}).get('cls_token') + special_eos = (tokenizer_config or {}).get('eos_token') + special_sep = (tokenizer_config or {}).get('sep_token') + if not special_bos and special_cls and tokenizer_config: + tokenizer_config['bos_token'] = special_bos = special_cls + if not special_eos and special_sep and tokenizer_config: + tokenizer_config['eos_token'] = special_eos = special_sep + if post_processor := tokenizer.get('post_processor'): + for processor in post_processor.get('processors', [post_processor]): + if processor.get('type') == 'RobertaProcessing': + self.add_special_token['bos'] = True + self.add_special_token['eos'] = True + self.add_special_token['sep'] = True + if not special_cls and tokenizer_config: + special_cls = processor.get('cls', [special_bos])[0] + tokenizer_config['cls_token'] = special_cls + if not special_sep and tokenizer_config: + special_sep = processor.get('sep', [special_eos])[0] + tokenizer_config['sep_token'] = special_sep + continue + # Crude parsing of TemplateProcessing to determine if BOS/SEP/EOS should be added + # Only works with simple templates, **will** get it wrong on unusual sequences + if processor.get('type') == 'TemplateProcessing': + tmpl_single = processor.get('single', []) + tmpl_pair = processor.get('pair', []) + special_first = None + special_last = None + if len(tmpl_single) > 1: + if special_first := tmpl_single[0].get('SpecialToken', {}).get('id'): + if not tokenizer_config: + special_bos = special_first + self.add_special_token['bos'] = True if special_first in (special_bos, special_cls) else False + if special_first not in (special_bos, special_cls): + logger.warning(f'Unknown leading special token {special_first!r} in TemplateProcessing') + if special_last := tmpl_single[-1].get('SpecialToken', {}).get('id'): + if not tokenizer_config: + special_eos = special_last + elif special_last != special_eos: + if 'eot' not in self.special_token_types: + self.special_token_types = tuple(self.special_token_types) + ('eot', ) + tokenizer_config['eot_token'] = special_eos + elif 'eom' not in self.special_token_types: + self.special_token_types = tuple(self.special_token_types) + ('eom', ) + tokenizer_config['eom_token'] = special_eos + else: + logger.warning(f'Overriding EOS token {special_eos!r} with {special_last!r} without EOT/EOM fallback!') + tokenizer_config['eos_token'] = special_eos = special_last + self.add_special_token['eos'] = True if special_last == special_eos else False + if special_last != special_eos: + logger.warning(f'Unknown trailing special token {special_last!r} in TemplateProcessing') + if tmpl_pair: + seq_start = 1 if special_first and tmpl_pair[0].get('SpecialToken', {}).get('id') == special_first else 0 + seq_stop = -1 if special_last and tmpl_pair[-1].get('SpecialToken', {}).get('id') == special_last else None + if (special_first and seq_start == 0) or (special_last and seq_stop is None): + logger.warning('TemplateProcessing leading/trailing special tokens do not match TemplateProcessing') + if tmpl_pair := tmpl_pair[slice(seq_start, seq_stop)]: + tmpl_a = tmpl_pair[0].get('Sequence', {}).get('id') + tmpl_b = tmpl_pair[-1].get('Sequence', {}).get('id') + if tmpl_a != 'A' or tmpl_b != 'B': + logger.warning(f'Unknown sequence {tmpl_a}...{tmpl_b} in TemplateProcessing') + # A [sep] [eos] B + if tmpl_a == 'A' and tmpl_b == 'B' and (tmpl_pair := tmpl_pair[1:-1]): + add_sep = False + if special_entry := tmpl_pair[0].get('SpecialToken', {}).get('id'): + if special_entry in (special_sep, special_eos) and not special_last: + add_sep = True + if special_entry not in (special_sep, special_eos): + logger.warning(f'Unknown separator token {special_entry!r} in TemplateProcessing') + else: + logger.warning(f'Unknown middle sequence {tmpl_pair[0]!r} in TemplateProcessing') + if len(tmpl_pair) == 2: + if special_entry := tmpl_pair[1].get('SpecialToken', {}).get('id'): + if special_entry in (special_sep, special_eos): + add_sep = True + if special_entry not in (special_sep, special_eos): + logger.warning(f'Unknown second separator token {special_entry!r} in TemplateProcessing') + else: + logger.warning(f'Unknown second middle sequence {tmpl_pair[1]!r} in TemplateProcessing') + self.add_special_token['sep'] = add_sep + if add_sep and not special_sep and tokenizer_config: + tokenizer_config['sep_token'] = special_eos + continue + if not tokenizer_config: return True - with open(tokenizer_config_file, encoding = 'utf-8') as f: - tokenizer_config = json.load(f) chat_template_alt = None - chat_template_file = path / 'chat_template.json' - if chat_template_file.is_file(): - with open(chat_template_file, encoding = 'utf-8') as f: + chat_template_json = path / 'chat_template.json' + chat_template_jinja = path / 'chat_template.jinja' + if chat_template_jinja.is_file(): + with open(chat_template_jinja, encoding = 'utf-8') as f: + chat_template_alt = f.read() + if additional_templates := list((path / 'additional_chat_templates').glob('*.jinja')): + chat_template_alt = [{'name': 'default', 'template': chat_template_alt}] + for template_path in additional_templates: + with open(template_path, encoding = 'utf-8') as fp: + chat_template_alt.append({'name': template_path.stem, 'template': fp.read()}) + elif chat_template_json.is_file(): + with open(chat_template_json, encoding = 'utf-8') as f: chat_template_alt = json.load(f).get('chat_template') chat_template = tokenizer_config.get('chat_template', chat_template_alt) if chat_template is None or isinstance(chat_template, (str, list)): @@ -302,6 +421,9 @@ class SentencePieceVocab(Vocab): name = "spm" def __init__(self, base_path: Path): + if SentencePieceProcessor is None: + raise RuntimeError("sentencepiece is not installed") + added_tokens: dict[str, int] = {} if (fname_tokenizer := base_path / 'tokenizer.model').exists(): # normal location @@ -490,3 +612,262 @@ def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: def __repr__(self) -> str: return f"" + + +class MistralTokenizerType(str, Enum): + spm = "spm" + tekken = "tekken" + + +# Copied from Transformers (Apache 2.0) +# https://github.com/huggingface/transformers/blob/main/src/transformers/convert_slow_tokenizer.py#L1544 + +def bytes_to_unicode() -> dict[int, str]: + """ + Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control + characters the bpe code barfs on. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab + if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for + decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup + tables between utf-8 bytes and unicode strings. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("¡"), ord("¬") + 1)) + + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs_str = [chr(n) for n in cs] + return dict(zip(bs, cs_str)) + + +class MistralVocab(Vocab): + tokenizer_model = "mistral" + name = "mistral" + + added_tokens_dict: dict[str, int] = {} + added_tokens_list: list[str] = [] + + def __init__(self, base_path: Path): + if not _mistral_common_installed: + raise ImportError( + "To use MistralVocab, please install the `mistral-common` package. " + "You can install it with `pip install mistral-common`." + ) + assert _filter_valid_tokenizer_files is not None, "mistral_common is not installed" + assert MistralTokenizer is not None, "mistral_common is not installed" + assert Tekkenizer is not None, "mistral_common is not installed" + + logger.info(f"Loading Mistral tokenizer from {base_path}") + + # Find the tokenizer files + all_files = [f.as_posix() for f in base_path.glob("**/*") if f.is_file()] + valid_tokenizer_files = _filter_valid_tokenizer_files(all_files) + + if len(valid_tokenizer_files) == 0: + raise ValueError(f"No tokenizer file found in the directory: {base_path}") + # If there are multiple tokenizer files, we use tekken.json if it exists, otherwise the versioned one. + if len(valid_tokenizer_files) > 1: + if "tekken.json" in valid_tokenizer_files: + tokenizer_file = "tekken.json" + else: + tokenizer_file = sorted(valid_tokenizer_files)[-1] + logger.warning( + f"Multiple tokenizer files found in {base_path}. Using {tokenizer_file}" + ) + else: + tokenizer_file = valid_tokenizer_files[0] + + self.tokenizer = MistralTokenizer.from_file( + base_path / tokenizer_file + ).instruct_tokenizer.tokenizer + self.tokenizer_type = ( + MistralTokenizerType.tekken + if isinstance(self.tokenizer, Tekkenizer) + else MistralTokenizerType.spm + ) + self.vocab_size = self.tokenizer.n_words + self.fname_tokenizer = base_path / tokenizer_file + self._name = ( + "mistral-" + self.tokenizer_type.value + "-" + self.tokenizer.version + ) + + @property + def tokenizer_name(self) -> str: + return self._name + + @property + def gguf_tokenizer_model(self) -> str: + return "llama" if self.tokenizer_type == MistralTokenizerType.spm else "gpt2" + + def _sentencepiece_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: + assert SentencePieceTokenizer is not None, "mistral_common is not installed" + assert isinstance(self.tokenizer, SentencePieceTokenizer), ( + f"Expected SentencePieceTokenizer, got {type(self.tokenizer)}" + ) + + for i in range(self.tokenizer._model.vocab_size()): + piece = self.tokenizer._model.IdToPiece(i) + text = piece.encode("utf-8") + score: float = self.tokenizer._model.GetScore(i) + + toktype = gguf.TokenType.NORMAL + if self.tokenizer._model.IsUnknown(i): + toktype = gguf.TokenType.UNKNOWN + if self.tokenizer._model.IsControl(i): + toktype = gguf.TokenType.CONTROL + + if self.tokenizer._model.IsUnused(i): + toktype = gguf.TokenType.UNUSED + if self.tokenizer._model.IsByte(i): + toktype = gguf.TokenType.BYTE + + yield text, score, toktype + + def _tekken_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: + assert Tekkenizer is not None, "mistral_common is not installed" + assert isinstance(self.tokenizer, Tekkenizer), ( + f"Expected Tekkenizer, got {type(self.tokenizer)}" + ) + + byte_encoder = bytes_to_unicode() + for token_id in range(self.tokenizer.num_special_tokens): + yield ( + self.tokenizer.id_to_piece(token_id).encode("utf-8"), + 0, + gguf.TokenType.CONTROL + ) + for token in self.tokenizer._tekken_token2id_nospecial: + yield ( + self.token_bytes_to_string(token, byte_encoder).encode("utf-8"), + 0, + gguf.TokenType.NORMAL, + ) + + def get_token_id(self, token: str) -> int: + assert SentencePieceTokenizer is not None and Tekkenizer is not None, "mistral_common is not installed" + if self.tokenizer_type == MistralTokenizerType.spm: + assert isinstance(self.tokenizer, SentencePieceTokenizer) + return self.tokenizer._vocab.index(token) + elif self.tokenizer_type == MistralTokenizerType.tekken: + assert isinstance(self.tokenizer, Tekkenizer) + return ( + self.tokenizer._vocab.index(token) + self.tokenizer.num_special_tokens + ) + else: + raise ValueError(f"Unknown tokenizer type: {self.tokenizer_type}") + + @property + def bos_id(self) -> int: + return self.tokenizer.bos_id + + @property + def eos_id(self) -> int: + return self.tokenizer.eos_id + + @property + def pad_id(self) -> int: + if self.tokenizer.pad_id == -1: + return self.eos_id + return self.tokenizer.pad_id + + @property + def unk_id(self) -> int: + return self.tokenizer.unk_id + + @property + def bos_token(self) -> str: + return self.tokenizer.id_to_piece(self.tokenizer.bos_id) + + @property + def eos_token(self) -> str: + return self.tokenizer.id_to_piece(self.tokenizer.eos_id) + + @property + def pad_token(self) -> str: + return self.tokenizer.id_to_piece(self.tokenizer.pad_id) + + @property + def unk_token(self) -> str: + return self.tokenizer.id_to_piece(self.tokenizer.unk_id) + + def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: + if self.tokenizer_type == MistralTokenizerType.spm: + yield from self._sentencepiece_tokens() + + elif self.tokenizer_type == MistralTokenizerType.tekken: + yield from self._tekken_tokens() + + else: + raise ValueError(f"Unknown tokenizer type: {self.tokenizer_type}") + + @staticmethod + def token_bytes_to_string(b, byte_encoder): + return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")]) + + def extract_vocab_merges_from_model(self): + # Adapted from Transformers (Apache 2.0) + # https://github.com/huggingface/transformers/blob/main/src/transformers/convert_slow_tokenizer.py + assert Tekkenizer is not None and isinstance(self.tokenizer, Tekkenizer), ( + f"Expected Tekkenizer, got {type(self.tokenizer)}" + ) + mergeable_ranks = self.tokenizer._model._mergeable_ranks + token_bytes_map = { + rank: token_bytes for token_bytes, rank in mergeable_ranks.items() + } + merge_pairs = [] + + # Sort vocab by rank to ensure correct merge order + for i in range(256, self.vocab_size - self.tokenizer.num_special_tokens): + merged_token = token_bytes_map[i] + local = [] + for j in range(1, len(merged_token)): + left = merged_token[:j] + right = merged_token[j:] + if ( + left in mergeable_ranks + and right in mergeable_ranks + and (left + right) in mergeable_ranks + ): + local.append((left, right, i)) + if not local: + raise ValueError( + f"Could not find valid merge for token at rank {i}: {merged_token.decode('latin-1')}" + ) + local = sorted( + local, + key=lambda x: (mergeable_ranks[x[0]], mergeable_ranks[x[1]]), + reverse=False, + ) + merge_pairs.extend(local) + merge_pairs = sorted(merge_pairs, key=lambda val: val[2], reverse=False) + + byte_encoder = bytes_to_unicode() + + decoded_merge_pairs = [ + [ + self.token_bytes_to_string(val[0], byte_encoder), + self.token_bytes_to_string(val[1], byte_encoder), + ] + for val in merge_pairs + ] + + merges = [ + " ".join( + [ + # ensure the spaces are properly encoded + "".join(chr(ord(c) + 256) if c == " " else c for c in part) + for part in pair + ] + ) + for pair in decoded_merge_pairs + ] + + return merges