[sync] vLLM 2026-07-15 — e26264f3e..647213129

.pre-commit-config.yaml

@@ -110,7 +110,7 @@ repos:
     name: Check SPDX headers
     entry: python tools/pre_commit/check_spdx_header.py
     language: python
-    types: [python]
+    types_or: [python, rust, proto]
     exclude: ^(reference/|aphrodite/vllm_flash_attn/|aphrodite/third_party/|tests/|examples/|benchmarks/)
   - id: check-root-lazy-imports
     name: Check root lazy imports

.sync/vllm-sha

@@ -1 +1 @@
-4c81772e8bdf9e37ab2b8adfd80e4308b618fa92
+6472131298e7b9ce81651cac90cc2dcc28963b55

CMakeLists.txt

@@ -1479,6 +1479,7 @@ if(APHRODITE_GPU_LANG STREQUAL "HIP")
   set(APHRODITE_ROCM_EXT_SRC
     "csrc/rocm/torch_bindings.cpp"
     "csrc/rocm/skinny_gemms.cu"
+    "csrc/rocm/skinny_gemms_int4.cu"
     "csrc/rocm/attention.cu")
 
   set(APHRODITE_ROCM_HAS_GFX1100 OFF)

aphrodite/_aiter_ops.py

@@ -2587,7 +2587,8 @@ def shuffle_weight_a16w4(
 
         Args:
             tensor: The input weight tensor to be shuffled.
-            layout: The block layout to use, defaults to (16, 4).
+            nLane: Number of lanes in the shuffle layout.
+            gate_up: Whether the weight is for w13 (True) or w2 (False).
 
         Returns:
             torch.Tensor: The shuffled tensor.
@@ -2823,7 +2824,6 @@ def mhc_pre(
             hc_sinkhorn_eps: sinkhorn epsilon
             hc_post_mult_value: post-mix multiplier value
             sinkhorn_repeat: number of sinkhorn iterations
-            n_splits: split-k factor;
 
         Returns:
             post_mix: shape (..., hc_mult), dtype torch.float32

aphrodite/_custom_ops.py

@@ -1960,7 +1960,11 @@ def scaled_fp4_quant(
     Args:
         input: The input tensor to be quantized to FP4
         input_global_scale: A scalar scaling factor for the entire tensor.
-        use_8x4_sf_layout: Whether to use the 8x4 or 128x4 layout for the scaling
+        is_sf_swizzled_layout: Whether to store the scaling factors in the
+            swizzled layout (default `True`).
+        backend: Quantization kernel backend to dispatch to. For `"trtllm"`
+            backends the 8x4 scale-factor layout is selected for small
+            batches (m <= 32) instead of the 128x4 layout.
         padded_n: Optional padded K dimension. When provided, the quantized
             output and scale tensors are allocated for ``padded_n``
 
@@ -2592,6 +2596,35 @@ def wvSplitK(a: torch.Tensor, b: torch.Tensor, cu_count: int, bias: torch.Tensor
     return torch.ops._rocm_C.wvSplitK(a, b, bias, cu_count)
 
 
+def wvSplitK_int4_g(
+    weight: torch.Tensor,
+    activation: torch.Tensor,
+    scale: torch.Tensor,
+    cu_count: int,
+    group_size: int,
+    zero_points: torch.Tensor | None = None,
+    bias: torch.Tensor | None = None,
+) -> torch.Tensor:
+    # The kernel is weight-major: weight is the packed int4 operand
+    # (in_a, [out_features, K/2]) and activation is in_b ([num_tokens, K]).
+    return torch.ops._rocm_C.wvSplitK_int4_g(weight, activation, scale, zero_points, bias, cu_count, group_size)
+
+
+if hasattr(torch.ops, "_rocm_C") and hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"):
+
+    @register_fake("_rocm_C::wvSplitK_int4_g")
+    def _wvSplitK_int4_g_fake(
+        in_a: torch.Tensor,
+        in_b: torch.Tensor,
+        in_scale: torch.Tensor,
+        in_zero_points: torch.Tensor | None,
+        in_bias: torch.Tensor | None,
+        CuCount: int,
+        group_size: int,
+    ) -> torch.Tensor:
+        return torch.empty((in_b.size(0), in_a.size(0)), dtype=in_b.dtype, device=in_b.device)
+
+
 def wvSplitKrc(a: torch.Tensor, b: torch.Tensor, cu_count: int, bias: torch.Tensor = None) -> torch.Tensor:
     return torch.ops._rocm_C.wvSplitKrc(a, b, bias, cu_count)
 
@@ -2611,8 +2644,13 @@ def wvSplitKQ(
 
 
 # moe
-def moe_sum(input: torch.Tensor, output: torch.Tensor):
-    torch.ops._moe_C.moe_sum(input, output)
+def moe_sum(
+    input: torch.Tensor,
+    output: torch.Tensor,
+    topk_ids: torch.Tensor | None = None,
+    expert_map: torch.Tensor | None = None,
+):
+    torch.ops._moe_C.moe_sum(input, output, topk_ids, expert_map)
 
 
 def moe_align_block_size(
@@ -3023,6 +3061,7 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
     q_out: torch.Tensor | None = None,
     index_q_out: torch.Tensor | None = None,
     kv_cache_dtype: str = "auto",
+    skip_index_branch: bool = False,
 ) -> None:
     """Fused MiniMax-M3 attention pre-processing (in-place).
 
@@ -3044,6 +3083,11 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
     instead of in place — folding the de-interleave into this kernel's store so
     callers skip a separate ``.contiguous()`` copy before the SM100 sparse
     attention's flat TMA descriptor.
+
+    When ``skip_index_branch`` is true, sparse rows still keep their packed
+    ``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v
+    branches and main KV cache. This is used by MiniMax-M3 index-topk reuse
+    layers that consume top-k block ids selected by an earlier sparse layer.
     """
     torch.ops._C.fused_minimax_m3_qknorm_rope_kv_insert(
         qkv,
@@ -3066,6 +3110,7 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
         q_out,
         index_q_out,
         kv_cache_dtype,
+        skip_index_branch,
     )
 
 
@@ -3803,6 +3848,40 @@ def fused_sigmoid_gating_delta_rule_update_cpu(
     )
 
 
+def fused_sigmoid_gating_delta_rule_update_spec_cpu(
+    A_log: torch.Tensor,
+    dt_bias: torch.Tensor,
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    a: torch.Tensor,
+    b: torch.Tensor,
+    initial_state_source: torch.Tensor,
+    spec_state_indices: torch.Tensor,
+    num_accepted_tokens: torch.Tensor,
+    cu_seqlens: torch.Tensor,
+    use_qk_l2norm_in_kernel: bool,
+    softplus_beta: float = 1.0,
+    softplus_threshold: float = 20.0,
+) -> torch.Tensor:
+    return torch.ops._C.fused_sigmoid_gating_delta_rule_update_spec_cpu(
+        A_log,
+        dt_bias,
+        q,
+        k,
+        v,
+        a,
+        b,
+        initial_state_source,
+        spec_state_indices,
+        num_accepted_tokens,
+        cu_seqlens,
+        use_qk_l2norm_in_kernel,
+        softplus_beta,
+        softplus_threshold,
+    )
+
+
 def fused_gdn_gating_cpu(
     A_log: torch.Tensor,
     a: torch.Tensor,

aphrodite/compilation/backends.py

@@ -1143,16 +1143,19 @@ def __call__(self, graph: fx.GraphModule, example_inputs: Sequence[Any]) -> Any:
         compilation_counter.num_graphs_seen += 1
         from .monitor import torch_compile_start_time
 
-        dynamo_time = time.perf_counter() - torch_compile_start_time
+        current_perf = time.perf_counter()
+        current_epoch = time.time()
+        dynamo_time = current_perf - torch_compile_start_time
         logger.info_once(
             "Dynamo bytecode transform time: %.2f s",
             dynamo_time,
         )
 
         # Record Dynamo time in tracing if available
-        start_time = int(torch_compile_start_time * 1e9)
+        real_start_time = current_epoch - dynamo_time
+        start_time_ns = int(real_start_time * 1e9)
         attributes = {"dynamo.time_seconds": dynamo_time}
-        instrument_manual("Dynamo bytecode transform", start_time, None, attributes)
+        instrument_manual("Dynamo bytecode transform", start_time_ns, None, attributes)
 
         # we control the compilation process, each instance can only be
         # called once

aphrodite/compilation/passes/fusion/allreduce_rms_fusion.py

@@ -50,6 +50,20 @@
 _IR_FUSED_ADD_RMS_NORM_OP = torch.ops.aphrodite_ir.fused_add_rms_norm.default
 
 
+def _view_nvfp4_scale_out_for_flashinfer(
+    scale_out: torch.Tensor,
+) -> torch.Tensor:
+    """View Aphrodite's packed NVFP4 scale buffer as FP8 for FlashInfer."""
+    return torch.ops.aten.view.dtype(scale_out, FP8_DTYPE)
+
+
+def _view_flashinfer_nvfp4_scale_out_as_int32(
+    scale_out: torch.Tensor,
+) -> torch.Tensor:
+    """View FlashInfer's NVFP4 scale buffer back as Aphrodite's int32 format."""
+    return torch.ops.aten.view.dtype(scale_out, torch.int32)
+
+
 def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
     """Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
     fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
@@ -194,8 +208,8 @@ def call_trtllm_fused_allreduce_norm(
         curr_device = current_platform.get_device_capability()
         device_capability = curr_device.to_int() if curr_device is not None else None
 
-        # Select workspace based on pattern: quant patterns use the
-        # trtllm quant workspace, non-quant patterns use the primary workspace.
+        # Select workspace based on pattern: quant patterns use the quant
+        # workspace, non-quant patterns use the primary workspace.
         is_quant_pattern = pattern_code in (
             ar_fusion_patterns.kARResidualRMSNormFP8Quant,
             ar_fusion_patterns.kARResidualRMSNormFP4Quant,
@@ -227,9 +241,9 @@ def call_trtllm_fused_allreduce_norm(
             residual_out = allreduce_in
 
         layout_code = None
-        # layout_code only supported by trtllm backend
-        if workspace.backend == "trtllm":
-            # in aphrodite we only support swizzled layout
+        # Aphrodite quant patterns use swizzled scale-factor layout. Non-quant
+        # patterns ignore layout_code.
+        if workspace.backend in ("trtllm", "mnnvl"):
             layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4
 
         flashinfer_comm.allreduce_fusion(
@@ -798,14 +812,15 @@ def replacement(
         ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
             residual = torch.zeros_like(input)
             result_rms = torch.empty_like(input)
+            output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale)
             assert flashinfer_comm is not None, "FlashInfer must be enabled"
             allreduce = auto_functionalized(
                 flashinfer_trtllm_fused_allreduce_norm,
                 allreduce_in=input,
                 residual=residual,
                 norm_out=result_rms,
                 quant_out=quant_result,
-                scale_out=output_scale,
+                scale_out=output_scale_fp8,
                 rms_gamma=weight,
                 rms_eps=self.epsilon,
                 # We don't use norm_out afterwards
@@ -815,7 +830,11 @@ def replacement(
             )
 
             # quant_out, allreduce_output, output_scale
-            return allreduce[4], allreduce[1], allreduce[5]
+            return (
+                allreduce[4],
+                allreduce[1],
+                _view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]),
+            )
 
         pm.register_replacement(
             pattern,
@@ -895,14 +914,15 @@ def replacement(
             weight: torch.Tensor,
             input_global_scale: torch.Tensor,
         ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+            output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale)
             assert flashinfer_comm is not None, "FlashInfer must be enabled"
             allreduce = auto_functionalized(
                 flashinfer_trtllm_fused_allreduce_norm,
                 allreduce_in=input,
                 residual=residual,
                 norm_out=None,
                 quant_out=quant_result,
-                scale_out=output_scale,
+                scale_out=output_scale_fp8,
                 rms_gamma=weight,
                 rms_eps=self.epsilon,
                 # We don't use norm_out afterwards
@@ -911,7 +931,11 @@ def replacement(
                 **self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
             )
             # quant_out, rms_norm_residual, output_scale
-            return allreduce[4], allreduce[2], allreduce[5]
+            return (
+                allreduce[4],
+                allreduce[2],
+                _view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]),
+            )
 
         pm.register_replacement(
             pattern,

aphrodite/config/attention.py

@@ -59,6 +59,11 @@ class AttentionConfig:
     use_non_causal: bool = False
     """Whether to use non-causal (bidirectional) attention."""
 
+    sparse_mla_force_mqa: bool = False
+    """Force sparse MLA to use forward_mqa for all requests, including prefill.
+    When False (default), pure prefill batches use forward_mha when implemented.
+    Set to True to always use the MQA path."""
+
     flex_attn_block_m: int | None = None
     """Triton kernel BLOCK_M tile size for flex attention.
     Must be a power of 2 >= 16. If None and APHRODITE_BATCH_INVARIANT=1,

aphrodite/config/model.py

@@ -20,7 +20,7 @@
     MMTensorIPC,
     MultiModalConfig,
 )
-from aphrodite.config.pooler import PoolerConfig
+from aphrodite.config.pooler import POOLER_CONFIG_LOG_FIELDS, PoolerConfig
 from aphrodite.config.quantization import QuantizationConfigArgs
 from aphrodite.config.scheduler import RunnerType
 from aphrodite.config.utils import config, getattr_iter
@@ -623,20 +623,30 @@ def __post_init__(
         if self.runner_type == "pooling":
             if self.pooler_config is None:
                 self.pooler_config = PoolerConfig()
+                pooler_config_sources: dict[str, str] = {}
+            else:
+                pooler_config_sources = {
+                    k: "user" for k in POOLER_CONFIG_LOG_FIELDS if getattr(self.pooler_config, k) is not None
+                }
 
             base_config = get_pooling_config(self.model, self.revision)
             if base_config is not None:
                 # Only set values that are not overridden by the user
                 for k, v in base_config.items():
                     if getattr(self.pooler_config, k) is None:
                         setattr(self.pooler_config, k, v)
+                        pooler_config_sources[k] = "sentence_transformers"
 
             default_seq_pooling_type = self._model_info.default_seq_pooling_type
             if self.pooler_config.seq_pooling_type is None:
                 self.pooler_config.seq_pooling_type = default_seq_pooling_type
+                pooler_config_sources["seq_pooling_type"] = "model_default"
             default_tok_pooling_type = self._model_info.default_tok_pooling_type
             if self.pooler_config.tok_pooling_type is None:
                 self.pooler_config.tok_pooling_type = default_tok_pooling_type
+                pooler_config_sources["tok_pooling_type"] = "model_default"
+            pooler_config_sources.setdefault("use_activation", "pooler_default")
+            self._pooler_config_sources = pooler_config_sources
 
         self.dtype: torch.dtype = _get_and_verify_dtype(
             self.model,
@@ -1611,21 +1621,16 @@ def head_dtype(self) -> torch.dtype:
         such as the lm_head in a generation model,
         or the score or classifier in a classification model.
 
-        `head_dtype` currently only supports pooling models.
-
-        - The pooling model defaults to using fp32 head, you can use
+        - Pooling models default to an fp32 head; use
           --hf-overrides '{"head_dtype": "model"}' to disable it.
+        - Generation models default to the model dtype; set
+          --hf-overrides '{"head_dtype": "float32"}' to run the lm_head in
+          fp32, which is required for RL training-inference consistency
+          (the trainer computes logits in fp32).
         """
 
         head_dtype = _get_head_dtype(config=self.hf_config, dtype=self.dtype, runner_type=self.runner_type)
 
-        if self.runner_type != "pooling" and head_dtype != self.dtype:
-            logger.warning_once(
-                "`head_dtype` currently only supports pooling models, fallback to model dtype [%s].",
-                self.dtype,
-            )
-            return self.dtype
-
         if head_dtype not in current_platform.supported_dtypes:
             logger.warning_once(
                 "The current platform does not support [%s] head dtype, fallback to model dtype [%s].",

aphrodite/config/parallel.py

@@ -628,7 +628,6 @@ def use_sequence_parallel_moe(self) -> bool:
             )
             and self.enable_expert_parallel
             and self.tensor_parallel_size > 1
-            and self.data_parallel_size > 1
         )
 
     @property

aphrodite/config/pooler.py

@@ -16,6 +16,12 @@
 TokenPoolingType = Literal["ALL", "STEP"]
 TOK_POOLING_TYPES: tuple[TokenPoolingType, ...] = get_args(TokenPoolingType)
 
+POOLER_CONFIG_LOG_FIELDS = (
+    "seq_pooling_type",
+    "tok_pooling_type",
+    "use_activation",
+)
+
 
 @config
 class PoolerConfig: