Skip to content

Commit 6929408

Browse files
[BugFix] fix qwen3.5+pcp accuray error (vllm-project#11195)
### What this PR does / why we need it? Fixing the precision issue of qwen3.5 with pcp integration 1. Uniformly use the `torch.ops._C_ascend.npu_causal_conv1d_custom` operator to prevent precision drift. 2. When integrating mtp, since `mix_qkv` has been split, `num_decode` cannot be directly obtained from the metadata and needs to be recalculated. [2026-06-29 11:59:55,012] [ais_bench] [INFO] Summarizing evaluation results... dataset version metric mode vllm-api-general-chat GPQA_diamond b1ed2c accuracy gen 86.00 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - vLLM version: v0.23.0 - vLLM main: vllm-project/vllm@a30addc --------- Signed-off-by: weiguihua2 <weiguihua2@huawei.com> Signed-off-by: hfadzxy <starmoon_zhang@163.com> Co-authored-by: hfadzxy <starmoon_zhang@163.com>
1 parent c9b9de8 commit 6929408

7 files changed

Lines changed: 67 additions & 162 deletions

File tree

.github/workflows/schedule_nightly_test_a3.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,9 @@ jobs:
295295
- name: Qwen3.5-397B-A17B-w8a8-mtp
296296
os: linux-aarch64-a3-16
297297
config_file_path: Qwen3.5-397B-A17B-W8A8-mtp-A3.yaml
298+
- name: Qwen3.5-397B-A17B-w8a8-mtp-longseq
299+
os: linux-aarch64-a3-16
300+
config_file_path: Qwen3.5-397B-A17B-w8a8-mtp-longseq-A3.yaml
298301
- name: MiniMax-M2.5-w8a8-QuaRot-A3
299302
os: linux-aarch64-a3-16
300303
config_file_path: MiniMax-M2.5-w8a8-QuaRot-A3.yaml

tests/e2e/nightly/single_node/models/configs/Qwen3.5-397B-A17B-w8a8-mtp-longseq-A3.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ test_cases:
2323
- "1"
2424
- "--prefill-context-parallel-size"
2525
- "2"
26+
- "--decode-context-parallel-size"
27+
- "4"
2628
- "--enable-expert-parallel"
2729
- "--port"
2830
- "$SERVER_PORT"
@@ -56,5 +58,5 @@ test_cases:
5658
num_prompts: 50
5759
max_out_len: 32768
5860
batch_size: 32
59-
baseline: 82
61+
baseline: 84
6062
threshold: 8

vllm_ascend/ops/gdn.py

Lines changed: 47 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
from vllm_ascend.ops.triton.fla.chunk import chunk_gated_delta_rule
4040
from vllm_ascend.ops.triton.fla.fused_qkvzba_split_reshape import fused_qkvzba_split_reshape_cat
4141
from vllm_ascend.ops.triton.fla.utils import clear_ssm_states
42-
from vllm_ascend.ops.triton.mamba.causal_conv1d import causal_conv1d_fn
42+
from vllm_ascend.ops.triton.mamba.causal_conv1d import extract_last_width
4343
from vllm_ascend.utils import weak_ref_tensors
4444

4545

@@ -505,21 +505,53 @@ def _forward_core(
505505
if attn_metadata.num_prefills > 0:
506506
if mixed_qkv_non_spec is not None:
507507
if get_pcp_group().world_size > 1:
508+
conv_weights_T = conv_weights.transpose(0, 1)
509+
activation_num = 1 if self.activation else 0
510+
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor
511+
(query_start_loc_opt, cache_indices_opt, initial_state_mode_opt) = (
512+
get_non_spec_causal_conv1d_host_args(attn_metadata)
513+
)
514+
width = conv_weights.shape[1]
515+
state_len = width - 1
516+
num_seqs = non_spec_query_start_loc.shape[0] - 1
517+
prefill_seq_offset = max(0, num_seqs - attn_metadata.num_prefills)
518+
prefill_cache_indices = non_spec_state_indices_tensor[prefill_seq_offset:]
508519
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
509-
has_initial_state = attn_metadata.has_initial_state
510-
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501
511-
conv_state = self_kv_cache[0].transpose(-1, -2)
512-
mixed_qkv_non_spec = causal_conv1d_fn(
513-
mixed_qkv_non_spec_T,
514-
conv_weights,
515-
self.conv1d.bias,
516-
activation=self.activation,
517-
conv_states=conv_state,
518-
has_initial_state=has_initial_state,
519-
cache_indices=non_spec_state_indices_tensor,
520-
query_start_loc=non_spec_query_start_loc,
521-
metadata=attn_metadata,
522-
).transpose(0, 1)
520+
last_width_prefill_x = extract_last_width(
521+
mixed_qkv_non_spec_T, non_spec_query_start_loc[prefill_seq_offset:], state_len
522+
)
523+
pcp_rank = get_pcp_group().rank_in_group
524+
all_last_width_prefill_x = get_pcp_group().all_gather(
525+
last_width_prefill_x.unsqueeze(0).contiguous(), 0
526+
)
527+
if pcp_rank > 0 and prefill_cache_indices.shape[0] > 0:
528+
self_kv_cache[0][prefill_cache_indices, :state_len, :] = all_last_width_prefill_x[
529+
pcp_rank - 1, ...
530+
].transpose(-1, -2)
531+
ism_list = list(initial_state_mode_opt)
532+
for i in range(prefill_seq_offset, len(ism_list)):
533+
ism_list[i] = 1
534+
initial_state_mode_opt = tuple(ism_list)
535+
mixed_qkv_non_spec_output = torch.empty_like(mixed_qkv_non_spec)
536+
torch.ops._C_ascend.npu_causal_conv1d_custom(
537+
mixed_qkv_non_spec_output,
538+
mixed_qkv_non_spec,
539+
conv_weights_T,
540+
conv_state=self_kv_cache[0],
541+
bias_opt=self.conv1d.bias,
542+
query_start_loc_opt=query_start_loc_opt,
543+
cache_indices_opt=cache_indices_opt,
544+
initial_state_mode_opt=initial_state_mode_opt,
545+
num_accepted_tokens_opt=[],
546+
activation_mode=activation_num,
547+
pad_slot_id=PAD_SLOT_ID,
548+
run_mode=0,
549+
)
550+
mixed_qkv_non_spec = mixed_qkv_non_spec_output
551+
if prefill_cache_indices.shape[0] > 0:
552+
self_kv_cache[0][prefill_cache_indices, :state_len, :] = all_last_width_prefill_x[
553+
-1, ...
554+
].transpose(-1, -2)
523555
else:
524556
conv_weights_T = conv_weights.transpose(0, 1)
525557
activation_num = 1 if self.activation else 0

vllm_ascend/ops/gdn_attn_builder.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ class GDNChunkedPrefillMetadata:
6060
final_chunk_indices_chunk64: torch.Tensor
6161
chunk_indices_large_block: torch.Tensor
6262
block_indices_cumsum: torch.Tensor
63+
num_decodes: int = 0
6364
_buffer_slot: object | None = None
6465

6566

@@ -368,9 +369,13 @@ def _build_chunked_prefill_metadata(
368369
cu_seqlens_cpu: torch.Tensor,
369370
slot: _GDNChunkedPrefillBufferSlot | None = None,
370371
) -> GDNChunkedPrefillMetadata:
372+
cu_seqlens_host = tuple(cu_seqlens_cpu.to(torch.int64).tolist())
373+
num_decodes = sum(
374+
1 for i in range(len(cu_seqlens_host) - 1) if 0 < cu_seqlens_host[i + 1] - cu_seqlens_host[i] <= 1
375+
)
371376
return GDNChunkedPrefillMetadata(
372377
cu_seqlens_cpu=cu_seqlens_cpu,
373-
cu_seqlens_host=tuple(cu_seqlens_cpu.to(torch.int64).tolist()),
378+
cu_seqlens_host=cu_seqlens_host,
374379
chunk_indices_chunk64_host=_build_chunk_indices_host(
375380
cu_seqlens_cpu,
376381
builder._ascend_gdn_chunk_size,
@@ -381,6 +386,7 @@ def _build_chunked_prefill_metadata(
381386
final_chunk_indices_chunk64=tensors["final_chunk_indices_chunk64"],
382387
chunk_indices_large_block=tensors["chunk_indices_large_block"],
383388
block_indices_cumsum=tensors["block_indices_cumsum"],
389+
num_decodes=num_decodes,
384390
_buffer_slot=slot,
385391
)
386392

vllm_ascend/ops/triton/fla/chunk.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ def chunk_gated_delta_rule_fwd(
116116
)
117117

118118
if get_pcp_group().world_size > 1:
119+
# When integrating mtp, since `mix_qkv` has been split, `num_decode`
120+
# cannot be directly obtained from the metadata and needs to be recalculated.
121+
actual_num_decodes = getattr(prebuilt_meta, "num_decodes", None)
122+
if actual_num_decodes is None:
123+
actual_num_decodes = num_decodes
119124
h_update = chunk_gated_delta_rule_fwd_hupdate(
120125
k=k,
121126
w=w,
@@ -125,7 +130,7 @@ def chunk_gated_delta_rule_fwd(
125130
chunk_indices=chunk_indices_chunk64,
126131
chunk_offsets=chunk_offsets_chunk64,
127132
update_chunk_offsets=update_chunk_offsets_chunk64,
128-
num_decodes=num_decodes,
133+
num_decodes=actual_num_decodes,
129134
)
130135
all_final_state = get_pcp_group().all_gather(final_state.unsqueeze(0), 0)
131136
final_chunk_indices = final_chunk_indices_chunk64

vllm_ascend/ops/triton/mamba/causal_conv1d.py

Lines changed: 0 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,7 @@
77
# and https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/mamba/ops/causal_conv1d.py
88
# mypy: ignore-errors
99

10-
from typing import Any
11-
1210
import torch
13-
import torch.nn.functional as F
14-
from vllm.distributed import get_pcp_group
15-
from vllm.forward_context import get_forward_context
1611
from vllm.triton_utils import HAS_TRITON, tl, triton
1712
from vllm.v1.attention.backends.utils import PAD_SLOT_ID # type: ignore
1813

@@ -26,143 +21,6 @@
2621
_pytorch_update = None
2722

2823

29-
def causal_conv1d_ref(
30-
x: torch.Tensor,
31-
weight: torch.Tensor,
32-
bias: torch.Tensor | None = None,
33-
initial_states: torch.Tensor | None = None,
34-
return_final_states: bool = False,
35-
final_states_out: torch.Tensor | None = None,
36-
activation: str | None = "silu",
37-
):
38-
"""
39-
x: (batch, dim, seqlen)
40-
weight: (dim, width)
41-
bias: (dim,)
42-
initial_states: (batch, dim, width - 1)
43-
final_states_out: (batch, dim, width - 1)
44-
out: (batch, dim, seqlen)
45-
"""
46-
if activation not in [None, "silu", "swish"]:
47-
raise NotImplementedError(f"causal_conv1d_ref activation must be None, silu, or swish, got {activation}")
48-
dtype_in = x.dtype
49-
x = x.to(weight.dtype)
50-
seqlen = x.shape[-1]
51-
dim, width = weight.shape
52-
53-
if initial_states is None:
54-
out = F.conv1d(x, weight.unsqueeze(1), bias, padding=width - 1, groups=dim)
55-
else:
56-
x = torch.cat([initial_states, x], dim=-1)
57-
out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim)
58-
out = out[..., :seqlen]
59-
60-
if return_final_states:
61-
final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to(dtype_in) # (batch, dim, width - 1)
62-
if final_states_out is not None:
63-
final_states_out.copy_(final_states)
64-
else:
65-
final_states_out = final_states
66-
out = (out if activation is None else F.silu(out)).to(dtype=dtype_in)
67-
return (out, None) if not return_final_states else (out, final_states_out)
68-
69-
70-
def causal_conv1d_fn(
71-
x: torch.Tensor,
72-
weight: torch.Tensor,
73-
bias: torch.Tensor | None = None,
74-
activation: str | None = "silu",
75-
conv_states: torch.Tensor | None = None,
76-
has_initial_state: torch.Tensor | None = None,
77-
cache_indices: torch.Tensor | None = None,
78-
query_start_loc: torch.Tensor | None = None,
79-
metadata: Any | None = None,
80-
pad_slot_id: int = PAD_SLOT_ID,
81-
):
82-
"""
83-
x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen
84-
sequences are concatenated from left to right for varlen
85-
weight: (dim, width)
86-
bias: (dim,)
87-
query_start_loc: (batch + 1) int32
88-
The cumulative sequence lengths of the sequences in
89-
the batch, used to index into sequence. prepended by 0.
90-
for example: query_start_loc = torch.Tensor([0,10,16,17]),
91-
x.shape=(dim,17)
92-
cache_indices: (batch) int32
93-
indicates the corresponding state index,
94-
like so: conv_state = conv_states[cache_indices[batch_id]]
95-
has_initial_state: (batch) bool
96-
indicates whether should the kernel take the current state as initial
97-
state for the calculations
98-
conv_states: (...,dim,width - 1) itype
99-
updated inplace if provided
100-
activation: either None or "silu" or "swish"
101-
pad_slot_id: int
102-
if cache_indices is passed, lets the kernel identify padded
103-
entries that will not be processed,
104-
for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
105-
in this case, the kernel will not process entries at
106-
indices 0 and 3
107-
out: (batch, dim, seqlen)
108-
"""
109-
forward_context = get_forward_context()
110-
num_prefills = 0
111-
attn_metadata = forward_context.attn_metadata
112-
if attn_metadata is not None and isinstance(attn_metadata, dict):
113-
attn_metadata = next(iter(attn_metadata.values()), None)
114-
if attn_metadata is not None:
115-
num_prefills = attn_metadata.num_prefills
116-
117-
if activation not in [None, "silu", "swish"]:
118-
raise NotImplementedError(f"causal_conv1d_fn: activation must be None, silu, or swish, got {activation}")
119-
if x.stride(-1) != 1:
120-
x = x.contiguous()
121-
bias = bias.contiguous() if bias is not None else None
122-
123-
out_ref = []
124-
out_ref_b = []
125-
seqlens = query_start_loc[1:] - query_start_loc[:-1]
126-
seqlens = seqlens.tolist()
127-
splits = torch.split(x, seqlens, dim=-1)
128-
width = weight.shape[1]
129-
state_len = width - 1
130-
prefill_seq_offset = max(0, len(seqlens) - num_prefills)
131-
last_width_prefill_x = extract_last_width(x, query_start_loc[prefill_seq_offset:], state_len)
132-
133-
pcp_rank = get_pcp_group().rank_in_group
134-
if get_pcp_group().world_size > 1:
135-
all_last_width_prefill_x = get_pcp_group().all_gather(last_width_prefill_x.unsqueeze(0).contiguous(), 0)
136-
if pcp_rank > 0:
137-
conv_states[cache_indices[prefill_seq_offset:], :, :state_len] = all_last_width_prefill_x[pcp_rank - 1, ...]
138-
139-
for i in range(len(seqlens)):
140-
x_s = splits[i]
141-
if cache_indices[i] == PAD_SLOT_ID:
142-
continue
143-
if pcp_rank == 0 and not bool(has_initial_state[i].item()):
144-
initial_states = None
145-
else:
146-
initial_states = conv_states[cache_indices[i]][..., : (width - 1)]
147-
out_ref_b.append(
148-
causal_conv1d_ref(
149-
x_s,
150-
weight,
151-
bias,
152-
activation=activation,
153-
return_final_states=True,
154-
final_states_out=conv_states[cache_indices[i]][..., : (width - 1)].unsqueeze(0),
155-
initial_states=initial_states,
156-
)
157-
)
158-
159-
if get_pcp_group().world_size > 1:
160-
conv_states[cache_indices[prefill_seq_offset:], :, :state_len] = all_last_width_prefill_x[-1, ...]
161-
out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=-1))
162-
out_ref_tensor = torch.cat(out_ref, dim=0)
163-
return out_ref_tensor
164-
165-
16624
def extract_last_width(x, start_loc, width):
16725
end_loc = start_loc[1:]
16826
offsets = torch.arange(width, device=x.device)

vllm_ascend/patch/worker/patch_triton.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,11 @@
77
from vllm_ascend.ops.triton.fla.chunk import chunk_gated_delta_rule
88
from vllm_ascend.ops.triton.fla.layernorm_guard import LayerNormFn
99
from vllm_ascend.ops.triton.fla.sigmoid_gating import fused_recurrent_gated_delta_rule_fwd_kernel
10-
from vllm_ascend.ops.triton.mamba.causal_conv1d import causal_conv1d_fn, causal_conv1d_update_npu
10+
from vllm_ascend.ops.triton.mamba.causal_conv1d import causal_conv1d_update_npu
1111

1212
triton.next_power_of_2 = next_power_of_2
1313

1414
vllm.model_executor.layers.mamba.ops.causal_conv1d.causal_conv1d_update = causal_conv1d_update_npu
15-
vllm.model_executor.layers.mamba.ops.causal_conv1d.causal_conv1d_fn = causal_conv1d_fn
1615
vllm.model_executor.layers.fla.ops.fused_recurrent.fused_recurrent_gated_delta_rule_fwd_kernel = (
1716
fused_recurrent_gated_delta_rule_fwd_kernel
1817
)

0 commit comments

Comments
 (0)