@@ -474,9 +474,7 @@ def token_dispatch(
474474 self ,
475475 token_dispatch_input : MoETokenDispatchInput ,
476476 ):
477- use_mxfp_quant = token_dispatch_input .quant .is_mxfp
478- with_quant = token_dispatch_input .quant .dispatch_with_quant
479- scale_type = token_dispatch_input .quant .get_scale_type
477+ with_quant = token_dispatch_input .quant .is_int_quant or token_dispatch_input .quant .is_fp8
480478 hidden_states = token_dispatch_input .hidden_states
481479 topk_weights = token_dispatch_input .topk_weights
482480 topk_ids = token_dispatch_input .topk_ids
@@ -494,9 +492,9 @@ def token_dispatch(
494492
495493 dynamic_scale_after_all2all = None
496494 if with_quant :
497- dst_type = token_dispatch_input .quant .get_dst_type
498- permutated_local_input_tokens , dynamic_scale = DeviceOperator .npu_dynamic_quant (
499- permutated_local_input_tokens , act_quant_type = dst_type , use_mxfp_quant = use_mxfp_quant
495+ dst_type = torch . float8_e4m3fn if token_dispatch_input .quant .is_fp8 else torch . int8
496+ permutated_local_input_tokens , dynamic_scale = torch_npu .npu_dynamic_quant (
497+ permutated_local_input_tokens , dst_type = dst_type
500498 )
501499 _ , dynamic_scale_after_all2all , permute2_ep_all_to_all_handle = async_all_to_all (
502500 dynamic_scale , output_splits , input_splits , self .ep_group
@@ -517,7 +515,6 @@ def token_dispatch(
517515 dynamic_scale_after_all2all ,
518516 global_input_tokens_local_experts_indices ,
519517 with_quant ,
520- scale_type ,
521518 )
522519 )
523520
@@ -633,43 +630,17 @@ def _preprocess(self, topk_ids: torch.Tensor):
633630 )
634631
635632 def _dispatch_postprocess (
636- self ,
637- global_input_tokens ,
638- dynamic_scale_after_all2all ,
639- global_input_tokens_local_experts_indices ,
640- with_quant ,
641- scale_type ,
633+ self , global_input_tokens , dynamic_scale_after_all2all , global_input_tokens_local_experts_indices , with_quant
642634 ):
643635 # Early return if no local experts or no tokens
644636 if self .num_local_experts <= 1 :
645637 return global_input_tokens , dynamic_scale_after_all2all , None
646638
647- assert global_input_tokens_local_experts_indices is not None , (
648- "global_input_tokens_local_experts_indices must be provided"
649- )
650-
651- experts_indices_2d_copy = global_input_tokens_local_experts_indices .reshape (
652- global_input_tokens_local_experts_indices .shape [0 ], 1
653- )
654-
655- if scale_type == torch .float8_e8m0fnu :
656- dynamic_scale_for_routing = dynamic_scale_after_all2all .view (torch .float8_e8m0fnu )
657- global_input_tokens , reversed_global_input_permutation_mapping , _ , routed_scale = (
658- torch_npu .npu_moe_init_routing_v2 (
659- global_input_tokens ,
660- experts_indices_2d_copy ,
661- scale = dynamic_scale_for_routing ,
662- active_num = experts_indices_2d_copy .shape [0 ],
663- expert_num = self .num_local_experts ,
664- expert_tokens_num_type = 1 ,
665- expert_tokens_num_flag = True ,
666- active_expert_range = [0 , self .num_local_experts ],
667- )
639+ # Handle quantized case
640+ if with_quant :
641+ assert global_input_tokens_local_experts_indices is not None , (
642+ "global_input_tokens_local_experts_indices must be provided"
668643 )
669- dynamic_scale_after_all2all = routed_scale .view (torch .uint8 )
670- experts_indices_2d_copy .untyped_storage ().resize_ (0 )
671- return global_input_tokens , dynamic_scale_after_all2all , reversed_global_input_permutation_mapping
672- elif with_quant :
673644 dynamic_scale_after_all2all , _ = torch_npu .npu_moe_token_permute (
674645 dynamic_scale_after_all2all .unsqueeze (- 1 ), global_input_tokens_local_experts_indices
675646 )
0 commit comments