Skip to content

Commit 176bbad

Browse files
paulyu12paulyu12
authored
[Feature][LoRA][MoE] Support LoRA with unquantified MoE models (vllm-project#10977)
### What this PR does / why we need it? By injecting lora w13/w2 delta logics into `unquant_apply_mlp`, now this pr makes vLLM Ascend support lora with unquantified MoE models, such as Qwen3-30B-A3B. `TP>1` and `ACLGraph mode` are supported simultaneously. Performance improvement will be considered in the near future. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? pytest -sv tests/e2e/pull_request/one_card/lora/test_olmoe_lora.py pytest -sv tests/e2e/pull_request/two_card/lora/test_qwen3moe_lora_tp.py Co-authored with @liuchenbing and @wangzhao-11a. - vLLM version: v0.23.0 - vLLM main: vllm-project/vllm@1f486d9 --------- Signed-off-by: paulyu12 <507435917@qq.com>
1 parent 9e0c6ba commit 176bbad

13 files changed

Lines changed: 572 additions & 4 deletions

File tree

‎.github/workflows/scripts/test_config.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,7 @@ estimated_times:
685685
tests/e2e/pull_request/one_card/lora/test_ilama_lora.py: 210
686686
tests/e2e/pull_request/one_card/lora/test_llama32_lora.py: 340
687687
tests/e2e/pull_request/one_card/lora/test_lora_with_spec_decode.py: 650
688+
tests/e2e/pull_request/one_card/lora/test_olmoe_lora.py: 400
688689
tests/e2e/pull_request/one_card/lora/test_qwen35_densemodel_lora.py: 360
689690
tests/e2e/pull_request/one_card/lora/test_qwen3_multi_loras.py: 170
690691
tests/e2e/pull_request/one_card/lora/test_qwen3_reranker_lora.py: 310
@@ -723,6 +724,7 @@ estimated_times:
723724
tests/e2e/pull_request/two_card/aclgraph/test_aclgraph_capture_replay.py: 20
724725
tests/e2e/pull_request/two_card/lora/test_ilama_lora_tp2.py: 140
725726
tests/e2e/pull_request/two_card/lora/test_llama32_lora_tp2.py: 470
727+
tests/e2e/pull_request/two_card/lora/test_qwen3moe_lora_tp.py: 600
726728
tests/e2e/pull_request/two_card/spec_decode/test_spec_decode.py: 760
727729
tests/e2e/pull_request/two_card/test_data_parallel.py: 540
728730
tests/e2e/pull_request/two_card/test_deepseek_multistream_moe.py: 150

‎docs/source/user_guide/feature_guide/lora.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ vllm serve meta-llama/Llama-2-7b \
2323
--lora-modules '{"name": "sql-lora", "path": "/path/to/lora", "base_model_name": "meta-llama/Llama-2-7b"}'
2424
```
2525

26-
## Custom LoRA Operators
26+
## Note
2727

28-
We have implemented LoRA-related AscendC operators, such as bgmv_shrink, bgmv_expand, sgmv_shrink and sgmv_expand. You can find them under the "csrc/kernels" directory of [vllm-ascend repo](https://github.com/vllm-project/vllm-ascend/tree/main/csrc/kernels).
28+
- We have implemented LoRA-related AscendC operators, such as bgmv_shrink, bgmv_expand, sgmv_shrink and sgmv_expand. You can find them under the "csrc/kernels" directory of [vllm-ascend repo](https://github.com/vllm-project/vllm-ascend/tree/main/csrc/kernels).
29+
30+
- You can enable LoRA with dense or mixture-of-experts(MoE) models now ([PR #10977](https://github.com/vllm-project/vllm-ascend/pull/10977)). However, we haven't support expert-parallel(EP) or quantification yet when you run MoE models with LoRA.

‎tests/e2e/conftest.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1901,6 +1901,16 @@ def qwen35_text_lora_files():
19011901
return snapshot_download(repo_id="vllm-ascend/qwen35-4b-text-only-sql-lora")
19021902

19031903

1904+
@pytest.fixture(scope="session")
1905+
def qwen3moe_lora_files():
1906+
return snapshot_download(repo_id="vllm-ascend/qwen3-moe-text2sql-spider")
1907+
1908+
1909+
@pytest.fixture(scope="session")
1910+
def olmoe_lora_files():
1911+
return snapshot_download(repo_id="vllm-ascend/olmoe-instruct-text2sql-spider")
1912+
1913+
19041914
def qwen_prompt(questions: list[str]) -> list[str]:
19051915
placeholder = "<|image_pad|>"
19061916
return [

‎tests/e2e/pull_request/one_card/lora/test_olmoe_lora.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
from collections.abc import Sequence
2+
3+
import vllm
4+
from vllm.lora.request import LoRARequest
5+
6+
MODEL_PATH = "allenai/OLMoE-1B-7B-0125-Instruct"
7+
8+
PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me. Do not return any additional explanation. Below is an instruction that describes a task, Write a response that appropriately completes the request.
9+
"
10+
##Instruction:
11+
candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
12+
Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
13+
The People_ID of candidate is the foreign key of People_ID of people.
14+
15+
16+
###Input:
17+
{context}
18+
19+
###Response:""" # noqa: E501
20+
21+
EXPECTED_LORA_OUTPUT = [
22+
"SELECT count(*) FROM candidate",
23+
"SELECT count(*) FROM candidate",
24+
"SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
25+
"SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
26+
]
27+
28+
EXPECTED_BASE_MODEL_OUTPUT = [
29+
"SELECT COUNT(Candidate_ID) FROM candidate",
30+
"SELECT COUNT(Candidate_ID) FROM candidate",
31+
"SELECT Candidate_ID, COUNT(*) as Total_Candidates\nFROM candidate\nINNER JOIN people ON candidate.People_ID = people.People_ID", # noqa: E501
32+
# There are multiple acceptable responses
33+
(
34+
"SELECT Candidate_ID, Poll_Source FROM candidate WHERE People_ID IN (SELECT People_ID FROM people) ORDER BY COUNT(*) DESC LIMIT 1", # noqa: E501
35+
"SELECT Candidate_ID, Poll_Source FROM candidate WHERE COUNT(People_ID) = (SELECT COUNT(People_ID) FROM people) ORDER BY Candidate_ID DESC LIMIT 1", # noqa: E501
36+
),
37+
]
38+
39+
40+
def _output_matches(generated: str, accepted: str | Sequence[str]) -> bool:
41+
if isinstance(accepted, str):
42+
accepted = (accepted,)
43+
return any(generated.startswith(s) for s in accepted)
44+
45+
46+
def generate_and_test(
47+
llm: vllm.LLM,
48+
lora_path: str,
49+
lora_id: list[int | None] | int | None,
50+
compare_lower: bool = False,
51+
) -> None:
52+
prompts = [
53+
PROMPT_TEMPLATE.format(context="How many candidates are there?"),
54+
PROMPT_TEMPLATE.format(context="Count the number of candidates."),
55+
PROMPT_TEMPLATE.format(
56+
context="Which poll resource provided the most number of candidate information?" # noqa: E501
57+
),
58+
PROMPT_TEMPLATE.format(context="Return the poll resource associated with the most candidates."),
59+
]
60+
61+
lora_request = None
62+
if isinstance(lora_id, int):
63+
lora_request = LoRARequest(str(lora_id), lora_id, lora_path)
64+
elif isinstance(lora_id, list):
65+
lora_request = [LoRARequest(str(i), i, lora_path) if i is not None else None for i in lora_id]
66+
67+
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
68+
outputs = llm.generate(prompts, sampling_params, lora_request=lora_request)
69+
# Print the outputs.
70+
generated_texts: list[str] = []
71+
for output in outputs:
72+
prompt = output.prompt
73+
generated_text = output.outputs[0].text.strip()
74+
generated_texts.append(generated_text)
75+
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
76+
77+
for i in range(len(EXPECTED_LORA_OUTPUT)):
78+
req_lora_id = lora_id[i] if isinstance(lora_id, list) else lora_id
79+
generated_text = generated_texts[i]
80+
expected_output = EXPECTED_LORA_OUTPUT[i] if req_lora_id is not None else EXPECTED_BASE_MODEL_OUTPUT[i]
81+
82+
if compare_lower:
83+
generated_text = generated_text.lower()
84+
if isinstance(expected_output, str):
85+
expected_output = (expected_output.lower(),)
86+
else:
87+
expected_output = tuple(s.lower() for s in expected_output)
88+
assert _output_matches(generated_text, expected_output), (
89+
f"Output {i}: {generated_text!r} does not match any of {expected_output!r}"
90+
)
91+
92+
93+
def test_olmoe_lora(olmoe_lora_files):
94+
# We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
95+
# Otherwise, the lora-test will fail due to CUDA OOM.
96+
llm = vllm.LLM(
97+
MODEL_PATH,
98+
max_model_len=1024,
99+
enable_lora=True,
100+
max_loras=4,
101+
enforce_eager=False,
102+
trust_remote_code=True,
103+
enable_chunked_prefill=True,
104+
)
105+
106+
generate_and_test(llm, olmoe_lora_files, lora_id=1)

‎tests/e2e/pull_request/two_card/lora/test_qwen3moe_lora_tp.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import vllm
2+
from vllm.lora.request import LoRARequest
3+
4+
MODEL_PATH = "Qwen/Qwen3-30B-A3B"
5+
6+
PROMPT_TEMPLATE = """<|im_start|>user
7+
I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
8+
"
9+
##Instruction:
10+
candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
11+
Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
12+
The People_ID of candidate is the foreign key of People_ID of people.
13+
14+
15+
###Input:
16+
{context}
17+
18+
###Response:<|im_end|>
19+
<|im_start|>assistant""" # noqa: E501
20+
21+
EXPECTED_LORA_OUTPUT = [
22+
"<think>\n\n</think>\n\nSELECT count(*) FROM candidate",
23+
"<think>\n\n</think>\n\nSELECT count(*) FROM candidate",
24+
"<think>\n\n</think>\n\nSELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
25+
"<think>\n\n</think>\n\nSELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
26+
]
27+
28+
29+
def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
30+
prompts = [
31+
PROMPT_TEMPLATE.format(context="How many candidates are there?"),
32+
PROMPT_TEMPLATE.format(context="Count the number of candidates."),
33+
PROMPT_TEMPLATE.format(
34+
context="Which poll resource provided the most number of candidate information?" # noqa: E501
35+
),
36+
PROMPT_TEMPLATE.format(context="Return the poll resource associated with the most candidates."),
37+
]
38+
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
39+
outputs = llm.generate(
40+
prompts,
41+
sampling_params,
42+
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
43+
)
44+
# Print the outputs.
45+
generated_texts: list[str] = []
46+
for output in outputs:
47+
prompt = output.prompt
48+
generated_text = output.outputs[0].text.strip()
49+
generated_texts.append(generated_text)
50+
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
51+
52+
for i in range(len(EXPECTED_LORA_OUTPUT)):
53+
assert generated_texts[i].startswith(EXPECTED_LORA_OUTPUT[i])
54+
55+
56+
def test_qwen3moe_lora(qwen3moe_lora_files):
57+
llm = vllm.LLM(
58+
MODEL_PATH,
59+
max_model_len=1024,
60+
enable_lora=True,
61+
max_loras=4,
62+
enforce_eager=True,
63+
trust_remote_code=True,
64+
enable_chunked_prefill=True,
65+
tensor_parallel_size=2,
66+
)
67+
68+
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)

0 commit comments

Comments
 (0)