diff --git a/.agents/skills/run-e2e/SKILL.md b/.agents/skills/run-e2e/SKILL.md index 0e4a162c..15c0b9e4 100644 --- a/.agents/skills/run-e2e/SKILL.md +++ b/.agents/skills/run-e2e/SKILL.md @@ -1,14 +1,18 @@ --- name: run-e2e -description: Use when the user asks to run, validate, or diagnose the AFD plugin's DeepSeek-V2-Lite end-to-end tests on GPU or Ascend NPU hardware, including PR-gate E2E, GSM8K-7 accuracy, graph, eager, DBO, or 2A2F scenarios. +description: Use when the user asks to run, validate, or diagnose the AFD plugin's DeepSeek-V2-Lite GPU/NPU or Qwen3 MoE GPU end-to-end tests, including PR-gate E2E, GSM8K-7 accuracy, graph, eager, DBO, or 2A2F scenarios. --- # Run AFD E2E Tests ## Scope -Run the four tests in -tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py: +Run one of the model suites: + +- `tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py` on GPU or NPU +- `tests/e2e/models/qwen3_moe/test_qwen3_moe.py` on GPU + +Each suite contains four default scenarios: - baseline-graph - afd-eager @@ -31,8 +35,8 @@ If both are available, ask which to use. If neither is available, stop. Before starting pytest, confirm: - AFD_E2E_DEVICES contains the device IDs required by test cases. -- AFD_GPU_E2E_MODEL / AFD_NPU_E2E_MODEL is set to a local path, or the - environment can download `deepseek-ai/DeepSeek-V2-Lite` via huggingface_hub. +- The backend model variable is set to a local path, or the environment can + download the selected suite's checkpoint via huggingface_hub. - The selected vllm command runs. - pytest, afd_plugin, lm_eval, datasets, and huggingface_hub are importable. - HF_HOME points to the Hugging Face cache used for GSM8K and model weights. @@ -56,7 +60,7 @@ export HF_HOME=/path/to/huggingface export AFD_E2E_BACKEND=gpu export AFD_E2E_DEVICES=0,1,2 # Optional if the model is already local: -# export AFD_GPU_E2E_MODEL=/path/to/DeepSeek-V2-Lite +# export AFD_GPU_E2E_MODEL=/path/to/model # Optional: export AFD_GPU_E2E_VLLM_BIN=/path/to/vllm ~~~ @@ -86,9 +90,17 @@ python -m pytest -q -s \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[baseline-graph]" ~~~ +For Qwen3 MoE on GPU, run: + +~~~bash +python -m pytest -q -s \ + tests/e2e/models/qwen3_moe/test_qwen3_moe.py +~~~ + Do not add backend markers or run scenarios in parallel; they share devices. -For the opt-in GPU/NPU 2A2F case, set four unique device IDs and run: +For the opt-in DeepSeek-V2-Lite GPU/NPU 2A2F case, set four unique device IDs +and run: ~~~bash export AFD_E2E_DEVICES=0,1,2,3 @@ -103,8 +115,9 @@ On cancellation, forward SIGTERM and allow over 90 seconds for cleanup. ### 5. Report -Success means 4 passed and 0 skipped. Report the failed scenario, first -actionable error, and cleanup status. Any skip is a gate failure. +Success means the selected suite reports 4 passed and 0 skipped. Report the +failed scenario, first actionable error, and cleanup status. Any skip is a +gate failure. ## Environment reference @@ -112,7 +125,7 @@ actionable error, and cleanup status. Any skip is a gate failure. |---|---|---| | AFD_E2E_BACKEND | both | yes: gpu or npu | | AFD_E2E_DEVICES | both | yes: exactly 3 unique IDs | -| AFD_GPU_E2E_MODEL | GPU | no; downloads DeepSeek-V2-Lite when unset | +| AFD_GPU_E2E_MODEL | GPU | no; downloads the selected suite's model when unset | | AFD_GPU_E2E_VLLM_BIN | GPU | no; defaults to vllm | | AFD_NPU_E2E_MODEL | NPU | no; downloads DeepSeek-V2-Lite when unset | | AFD_NPU_E2E_VLLM_BIN | NPU | no; defaults to vllm | diff --git a/docs/design/module/e2e_testing.md b/docs/design/module/e2e_testing.md index 2323e3ec..3e3247f6 100644 --- a/docs/design/module/e2e_testing.md +++ b/docs/design/module/e2e_testing.md @@ -25,6 +25,7 @@ validation_paths: - "tests/unit/test_e2e_process_utils.py" - "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py" - "tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py" + - "tests/e2e/models/qwen3_moe/test_qwen3_moe.py" upstream_refs: - "vLLM 0.26.0 serving and shutdown interfaces" - "lm-evaluation-harness GSM8K task and local-completions API" @@ -32,8 +33,9 @@ upstream_refs: verified_platform_refs: - "CUDA DeepSeek-V2-Lite" - "Ascend NPU DeepSeek-V2-Lite" + - "CUDA Qwen3 MoE" related_issues: [] -last_reviewed: 2026-08-12 +last_reviewed: 2026-08-13 --- # E2E testing @@ -139,5 +141,5 @@ of 1319. 7. Update `tests/e2e/README.md`. Update the `run-e2e` skill only when commands, variables, prerequisites, or the default suite change. -Before adding a second model, extract the shared environment, device, and -runner-subprocess logic from `test_deepseek_v2_lite.py`. +Model entrypoints reuse the subprocess and download helpers in +`tests/conftest.py`. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index fe9d88fa..f43eee6d 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,7 +1,7 @@ # End-to-End Tests -These tests validate DeepSeek-V2-Lite on real GPU or Ascend NPU hardware. -The default gate runs four scenarios: +These tests validate DeepSeek-V2-Lite on real GPU or Ascend NPU hardware and +Qwen3 MoE on real GPU hardware. Each default gate runs four scenarios: - `baseline-graph` - `afd-eager` @@ -27,9 +27,9 @@ Run from the repository root. The environment needs `vllm`, `pytest`, `afd_plugin`, `lm_eval`, `datasets`, and `huggingface_hub`. NPU also needs `torch_npu`. -The test downloads/caches `openai/gsm8k` and -`deepseek-ai/DeepSeek-V2-Lite` when the backend model env var is unset. Point -`HF_HOME` at a persistent cache if you want to reuse downloads across runs. +The selected test downloads/caches `openai/gsm8k` and its Hugging Face model +when the backend model env var is unset. Point `HF_HOME` at a persistent cache +if you want to reuse downloads across runs. GPU: @@ -38,7 +38,7 @@ export HF_HOME=/path/to/huggingface export AFD_E2E_BACKEND=gpu # Optional: export AFD_E2E_DEVICES=0,1,2 # Optional if the model is already local: -# export AFD_GPU_E2E_MODEL=/path/to/DeepSeek-V2-Lite +# export AFD_GPU_E2E_MODEL=/path/to/model ``` NPU: @@ -51,14 +51,19 @@ export AFD_E2E_BACKEND=npu # export AFD_NPU_E2E_MODEL=/path/to/DeepSeek-V2-Lite ``` -Then run: +Then run the selected model suite: ```bash +# DeepSeek-V2-Lite python -m pytest -q -s \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-eager]" \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph]" \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph-dbo]" \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[baseline-graph]" + +# Qwen3 MoE +python -m pytest -q -s \ + tests/e2e/models/qwen3_moe/test_qwen3_moe.py ``` Success means 4 passed and 0 skipped. @@ -78,8 +83,13 @@ For the weekly full GSM8K test, run only `afd-graph-dbo`: ```bash export AFD_GSM8K_LIMIT=all +# DeepSeek-V2-Lite python -m pytest -q -s \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph-dbo]" + +# Qwen3 MoE +python -m pytest -q -s \ + "tests/e2e/models/qwen3_moe/test_qwen3_moe.py::test_qwen3_moe[afd-graph-dbo]" ``` This evaluates all 1319 GSM8K test samples. Without `AFD_GSM8K_LIMIT`, each @@ -91,7 +101,8 @@ The repository includes the [`run-e2e`](../../.agents/skills/run-e2e/SKILL.md) skill. Open the repository in Codex and ask, for example: ```text -Use run-e2e to run the GPU E2E tests with HF_HOME /data/huggingface. +Use run-e2e to run the Qwen3 MoE GPU E2E tests with HF_HOME +/data/huggingface. ``` Provide `HF_HOME` and `AFD_E2E_BACKEND`. `AFD_E2E_DEVICES` is optional; when diff --git a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py new file mode 100644 index 00000000..131c0f7e --- /dev/null +++ b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CUDA Qwen3 MoE E2E scenarios.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +from tests.conftest import download_dataset, download_model, run_runner + +GSM8K_DATASET_ID = "openai/gsm8k" +GSM8K_DATASET_CONFIG = "main" +QWEN3_MOE_REPO_ID = "Qwen/Qwen3-30B-A3B" +QWEN3_MOE_MAX_MODEL_LEN = 4096 +SCENARIOS = ( + "baseline-graph", + "afd-eager", + "afd-graph", + "afd-graph-dbo", +) + + +def _required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} must be set") + return value + + +def prepare_e2e_assets() -> None: + """Ensure GSM8K and Qwen3-30B-A3B are available for the runner.""" + download_dataset(GSM8K_DATASET_ID, GSM8K_DATASET_CONFIG) + + if _required_env("AFD_E2E_BACKEND") != "gpu": + raise RuntimeError("Qwen3 MoE E2E supports only the 'gpu' backend") + + existing = os.environ.get("AFD_GPU_E2E_MODEL") + if existing: + print( + f"[e2e] Using existing model path {Path(existing).expanduser()}", + flush=True, + ) + return + + os.environ["AFD_GPU_E2E_MODEL"] = str(download_model(QWEN3_MOE_REPO_ID)) + + +def build_runner_command(scenario: str, gsm8k_output_path: Path) -> list[str]: + backend = _required_env("AFD_E2E_BACKEND") + if backend != "gpu": + raise RuntimeError("Qwen3 MoE E2E supports only the 'gpu' backend") + + env_devices = os.environ.get("AFD_E2E_DEVICES") + devices = ( + [item.strip() for item in env_devices.split(",") if item.strip()] + if env_devices + else ["0", "1", "2"] + ) + attention_devices = devices[:2] + ffn_devices = devices[2:] + + command = [ + sys.executable, + "-m", + "tests.e2e.runner", + "--model", + _required_env("AFD_GPU_E2E_MODEL"), + "--vllm-bin", + os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm"), + "--device-backend", + backend, + f"--common-vllm-arg=--max-model-len={QWEN3_MOE_MAX_MODEL_LEN}", + "--attention-devices", + ",".join(attention_devices), + ] + if scenario != "baseline-graph": + command.extend(["--ffn-devices", ",".join(ffn_devices)]) + command.extend( + [ + "--scenario", + scenario, + "--gsm8k-output-path", + str(gsm8k_output_path), + "--served-model-name-prefix", + "qwen3-moe-afd", + ], + ) + return command + + +@pytest.fixture(scope="module", autouse=True) +def _prepare_e2e_assets() -> None: + """Prepare shared E2E assets once for this test module.""" + prepare_e2e_assets() + + +@pytest.mark.e2e +@pytest.mark.parametrize("scenario", SCENARIOS, ids=SCENARIOS) +def test_qwen3_moe(scenario: str, tmp_path: Path) -> None: + command = build_runner_command(scenario, tmp_path / scenario) + run_runner(command) diff --git a/tests/unit/test_e2e_runner.py b/tests/unit/test_e2e_runner.py index 3010fc5d..63d5093d 100644 --- a/tests/unit/test_e2e_runner.py +++ b/tests/unit/test_e2e_runner.py @@ -16,6 +16,7 @@ from tests.e2e.models.deepseek_v2_lite import ( test_deepseek_v2_lite as deepseek_v2_lite_e2e, ) +from tests.e2e.models.qwen3_moe import test_qwen3_moe as qwen3_moe_e2e def test_baseline_entrypoint_uses_two_devices(monkeypatch, tmp_path): @@ -47,6 +48,28 @@ def test_deepseek_v2_lite_entrypoint_limits_context( assert "--common-vllm-arg=--max-model-len=4096" in command +def test_qwen3_moe_baseline_entrypoint_uses_two_devices(monkeypatch, tmp_path): + monkeypatch.setenv("AFD_E2E_BACKEND", "gpu") + monkeypatch.setenv("AFD_E2E_DEVICES", "2,4,6") + monkeypatch.setenv("AFD_GPU_E2E_MODEL", "model") + + command = qwen3_moe_e2e.build_runner_command("baseline-graph", tmp_path) + + assert command[command.index("--attention-devices") + 1] == "2,4" + assert "--ffn-devices" not in command + + +@pytest.mark.parametrize("scenario", qwen3_moe_e2e.SCENARIOS) +def test_qwen3_moe_entrypoint_limits_context(monkeypatch, tmp_path, scenario): + monkeypatch.setenv("AFD_E2E_BACKEND", "gpu") + monkeypatch.setenv("AFD_GPU_E2E_MODEL", "model") + monkeypatch.delenv("AFD_E2E_DEVICES", raising=False) + + command = qwen3_moe_e2e.build_runner_command(scenario, tmp_path) + + assert "--common-vllm-arg=--max-model-len=4096" in command + + def test_run_runner_forwards_cancellation_and_reaps(monkeypatch): command = [sys.executable, "-m", "tests.e2e.runner"] previous_handlers = {