Reuse LLM_FFN_SWIGLU_OAI_MOE

Author: timkhronos

src/llama-graph.cpp

@@ -1408,7 +1408,7 @@ ggml_tensor * llm_graph_context::build_ffn(
                 cur = ggml_swiglu(ctx0, cur);
                 cb(cur, "ffn_swiglu", il);
             } break;
-        case LLM_FFN_SWIGLU_OAI:
+        case LLM_FFN_SWIGLU_OAI_MOE:
             if (gate && type_gate == LLM_FFN_PAR) {
                 //Same constants as LLM_FFN_SWIGLU_OAI_MOE
                 const float alpha = 1.702f;

src/llama-graph.h

@@ -45,7 +45,6 @@ enum llm_ffn_op_type : int {
     LLM_FFN_SWIGLU,
     LLM_FFN_GEGLU,
     LLM_FFN_REGLU,
-    LLM_FFN_SWIGLU_OAI,
     LLM_FFN_SWIGLU_OAI_MOE,
 };
 

src/models/minimax-m3.cpp

@@ -570,7 +570,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                     model.layers[il].ffn_gate, NULL, NULL,
                     model.layers[il].ffn_down, NULL, NULL,
                     NULL,
-                    LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il);
+                    LLM_FFN_SWIGLU_OAI_MOE, LLM_FFN_PAR, il);
             cb(cur, "ffn_out", il);
         } else {
             // routed experts (swigluoai MoE)
@@ -593,7 +593,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                     model.layers[il].ffn_gate_shexp, NULL, NULL,
                     model.layers[il].ffn_down_shexp, NULL, NULL,
                     NULL,
-                    LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il);
+                    LLM_FFN_SWIGLU_OAI_MOE, LLM_FFN_PAR, il);
             cb(ffn_shexp, "ffn_shexp", il);
 
             cur = ggml_add(ctx0, moe_out, ffn_shexp);