From 94460c44ea59f62c50d61825291f38de793d7284 Mon Sep 17 00:00:00 2001 From: ShwStone Date: Thu, 13 Aug 2026 10:25:17 +0800 Subject: [PATCH 1/3] test(npu): fix v0.26 unit test fixtures and isolation Repair 28 NPU unit tests that failed on real hardware while CPU CI silently skipped them. All failures were test-side. - Stale v0.26 fixtures: use_v2_model_runner, afd_config/model_config, _ffn_loop_error, tensor-typed dp_metadata, read-only use_dcp -> dcp_size, required initialize_attn_backend kv_cache_config. - Cross-file pollution: pre-import the real module before the fake sys.modules swap so teardown has an original to restore. - Fake-torch leak: replace the string-typed fake torch with _FakeIntVector. Co-Authored-By: Claude --- .../npu/NPU_UNIT_TEST_FIX_2026-08-12.zh-CN.md | 97 +++++++++++++++++++ .../connectors/test_async_cam_connector.py | 20 +++- tests/unit/v1/worker/test_ffn_model_runner.py | 3 + tests/unit/v1/worker/test_npu_mla_graph.py | 21 ++++ tests/unit/v1/worker/test_npu_runtime.py | 30 ++++-- 5 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 docs/npu/NPU_UNIT_TEST_FIX_2026-08-12.zh-CN.md diff --git a/docs/npu/NPU_UNIT_TEST_FIX_2026-08-12.zh-CN.md b/docs/npu/NPU_UNIT_TEST_FIX_2026-08-12.zh-CN.md new file mode 100644 index 00000000..a1b030c6 --- /dev/null +++ b/docs/npu/NPU_UNIT_TEST_FIX_2026-08-12.zh-CN.md @@ -0,0 +1,97 @@ +# NPU 单元测试修复报告（2026-08-12） + +## 背景 + +vLLM 0.26 升级（c88babf）后，NPU 相关单元测试在真实 NPU 环境上大量失败（28 个），但 CPU-only CI 因 `pytest.importorskip("vllm"/"vllm_ascend")` 将这些模块整体跳过，失败长期不可见。本次在 afd-v026-1（Ascend A3）上将全量套件修到全绿。 + +## 最终结果 + +``` +python3 -m pytest -q tests/unit -m "not gpu and not vllm_runtime" +→ 全绿（~450 passed, 2 skipped, rc=0） +``` + +修复前 28 个失败，分布于： + +- `tests/unit/v1/worker/test_npu_runtime.py`（17） +- `tests/unit/v1/worker/test_ffn_model_runner.py`（10） +- `tests/unit/connectors/test_async_cam_connector.py`（1） + +**结论：28 个失败全部是测试侧问题（陈旧夹具/测试污染/环境 import 顺序），未发现插件代码 bug。** 运行时正确性由双机 GSM8K 实证（token-split 全量 0.9522）。 + +## 三层问题与修复 + +### 第一层：vllm_ascend 固有 circular import（挡住测试收集） + +vllm-ascend `80d8c194f`（v0.26 基线）存在固有循环导入： + +``` +device/device_op.py:32 → vllm_ascend.ops/__init__ → fused_moe/fused_moe.py + → fused_moe/experts_selector.py:25 → from vllm_ascend.device.device_op import DeviceOperator + （此时 device_op 只初始化了一半）→ ImportError +``` + +真实 `vllm serve` 的 import 顺序先加载 `vllm_ascend.ops`，天然规避；pytest 收集顺序则会触发。 + +**修复**：`tests/conftest.py` 顶部预导入（CPU CI 无此包则静默跳过）： + +```python +try: + import vllm_ascend.ops # noqa: F401 +except Exception: + pass +``` + +修复后不再需要任何手动 workaround，裸 pytest 直接可跑。 + +### 第二层：v0.26 升级后的陈旧测试夹具（26 个失败） + +| 失败模式 | 根因 | 修复 | +|---|---|---| +| `SimpleNamespace has no attribute 'use_v2_model_runner'` ×10 | vllm 0.26 的 `VllmConfig` 新字段，vllm_ascend `platform.set_additional_forward_context` 必读 | 测试夹具的 vllm_config 补 `use_v2_model_runner=False` | +| `AFDNPUFFNModelRunner has no attribute 'afd_config'` ×9 | 测试用 `object.__new__` 绕过 `__init__`，缺 `afd_config`/`model_config`（`_ffn_layer_indices` 要读） | 共享 helper `_new_ffn_runner()` 补齐（生产配置 `compute_gate_on_attention=True` + 全 MoE hf_config） | +| `AFDNPUFFNWorker has no attribute '_ffn_loop_error'` ×2 | 同上，`__init__` 才初始化该属性 | `_new_ffn_worker()` 补 `_ffn_loop_error=None` | +| `'list' object has no attribute 'tolist'` | mock dp_metadata 用 plain list 冒充 tensor（代码 `_dp_metadata_debug_key` 调 `.tolist()`） | 改 `torch.tensor([...])`（注意这些测试的 torch 是测试函数内局部 `importorskip` 绑定，需补绑定） | +| `use_dcp property has no setter` | vllm-ascend 0.26 把 `use_dcp` 改成只读 property（`dcp_size > 1` 派生） | 测试改设底层 `runner.dcp_size = 1` | +| `initialize_attn_backend() missing 'kv_cache_config'` | 上游 0.26 签名必填该参数（插件 override 与上游一致） | 测试调用改为 `initialize_attn_backend(None)` | +| `assert None is True` | `_warmup_and_capture` 返回 None，测试断言了不存在的返回值 | 删除多余断言（后续行为断言已覆盖测试意图） | +| `Invalid device string: 'int64'` | connector 测试把 async_cam 模块的 torch 换成假 torch（`int64="int64"` 字符串），却又往 states 里塞真 tensor，代码 `.to(torch.int64)` 变成真 tensor `.to("int64")` | 新增 `_FakeIntVector`（支持 `.to().sum().item()` 链）替换真 tensor | + +### 第三层：跨文件测试污染（1 个，最隐蔽） + +**现象**：`test_npu_runtime.py::test_npu_create_ascend_forward_context_marks_current_ubatch` 单跑通过、单文件跑通过、全量跑失败（child metadata 变成 parent 原值，`build_ubatch_afd_metadata` 未生效）。 + +**根因**：`test_npu_mla_graph.py::_load_forward_context_module` 往 `sys.modules` 塞入假的 `afd_plugin.v1.worker.ubatch_wrapper`（identity 版 `build_ubatch_afd_metadata`），再 reload `forward_context`。当真实模块此前**从未被 import** 时，monkeypatch teardown 没有原始对象可恢复，假绑定的 forward_context 经**父包属性绑定**泄漏给后续测试。 + +**验证方法**：二分定位（逐文件→逐测试），再用探针确认污染后 `fm.build_ubatch_afd_metadata` 指向假 lambda；预 import 真模块后污染消失。 + +**修复**：新增 `_preimport_real_module()`——装假模块前先 import 真身，teardown 才有真身可恢复；三个 `_load_*_module` helper 全部加上，try/except 保护无 vllm 的 CI 环境。 + +```python +def _preimport_real_module(module_name: str) -> None: + try: + importlib.import_module(module_name) + except Exception: + pass +``` + +## 改动清单 + +| 文件 | 改动 | +|---|---| +| `tests/conftest.py` | +11：预导入 vllm_ascend.ops 绕 circular import | +| `tests/unit/v1/worker/test_npu_runtime.py` | 夹具修复（afd_config/model_config/_ffn_loop_error/torch tensor 化/dcp_size/kv_cache_config/删无效断言） | +| `tests/unit/v1/worker/test_ffn_model_runner.py` | +3：vllm_config 补 use_v2_model_runner | +| `tests/unit/v1/worker/test_npu_mla_graph.py` | +21：_preimport_real_module 防泄漏 | +| `tests/unit/connectors/test_async_cam_connector.py` | +20：_FakeIntVector | + +另：recipe 启动脚本参数化（本次 GSM8K 双机部署使用）： +- `GPU_MEM_UTIL`（默认 0.90）——0.9 留给 HCCL_BUFFSIZE=4096 的通信池（实测 8.6GB）不够，0.8 可行 +- `ASYNC_MOE_UBATCHING`（默认 true）——跑 no-ubatch 对照用 + +## 经验教训 + +1. **CPU CI 的 importorskip 会让 NPU 单测静默腐烂**——涉及 vllm/vllm_ascend 的测试只有上 NPU pod 才真正执行，应定期在 pod 跑全量。 +2. **monkeypatch + importlib.reload 组合对"未被 import 过的模块"清理不干净**（包属性绑定泄漏）；sys.modules 换假模块的 helper 必须先 pre-import 真身。 +3. **判读测试失败先孤立复现**：单跑挂 = 夹具/代码问题；单跑过、全量挂 = 测试污染。 +4. mock 的字段类型要和真对象对齐（list vs tensor 的 `.tolist()`），0.26 升级把多个属性改成 property/必填参数，夹具要同步。 diff --git a/tests/unit/connectors/test_async_cam_connector.py b/tests/unit/connectors/test_async_cam_connector.py index 8e4102a2..0e64b530 100644 --- a/tests/unit/connectors/test_async_cam_connector.py +++ b/tests/unit/connectors/test_async_cam_connector.py @@ -39,6 +39,24 @@ def item(self): return self._value +class _FakeIntVector: + """Stands in for a token-count tensor in fake-torch tests. + + Supports the ``.to(dtype).sum().item()`` chain the connector applies to + ``group_list``; ``.to()`` is a no-op so the fake torch's string dtypes + cannot leak into a real tensor API. + """ + + def __init__(self, values): + self._values = [int(value) for value in values] + + def to(self, _dtype): + return self + + def sum(self): + return _FakeScalar(sum(self._values)) + + class _FakeTensorLike: def __init__(self, name, *, shape=None, device="npu:0"): self.name = name @@ -585,7 +603,7 @@ def fake_recv_attn_output(*, stage_idx, layer_idx, batch_size, ubatch_idx): _FakeScalar(7), ], expert_token_nums_shared=[_FakeScalar(5)], - group_list=torch.zeros(8, dtype=torch.int64), + group_list=_FakeIntVector([0] * 8), expand_x_shared=_FakeTensorLike("shared-hidden"), dynamic_scales_shared=_FakeTensorLike("shared-scales"), ) diff --git a/tests/unit/v1/worker/test_ffn_model_runner.py b/tests/unit/v1/worker/test_ffn_model_runner.py index bf018e01..19b2d4e1 100644 --- a/tests/unit/v1/worker/test_ffn_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_model_runner.py @@ -121,6 +121,9 @@ def _payload(hidden_states, metadata): def _runner_with_connector_and_model(model, *, num_layers=1): runner = object.__new__(GPUFFNModelRunner) runner.vllm_config = SimpleNamespace( + # vLLM 0.26 VllmConfig field read by the Ascend platform's + # set_additional_forward_context hook on NPU test environments. + use_v2_model_runner=False, parallel_config=SimpleNamespace( data_parallel_size=1, is_moe_model=True, diff --git a/tests/unit/v1/worker/test_npu_mla_graph.py b/tests/unit/v1/worker/test_npu_mla_graph.py index a6dba799..5ef4d035 100644 --- a/tests/unit/v1/worker/test_npu_mla_graph.py +++ b/tests/unit/v1/worker/test_npu_mla_graph.py @@ -22,7 +22,25 @@ def _reload_module( return importlib.import_module(module_name) +def _preimport_real_module(module_name: str) -> None: + """Import the real module before the fake sys.modules swap. + + ``_reload_module`` only restores state monkeypatch recorded: when the real + module was never imported, teardown has no original sys.modules entry or + parent-package attribute to restore, and the fresh fake-bound module leaks + into later tests through the package attribute. Pre-importing guarantees a + genuine original exists to restore. On environments without the real + dependencies (CPU-only CI) the import fails and the fake path is used as + before. + """ + try: + importlib.import_module(module_name) + except Exception: + pass + + def _load_mla_graph_module(monkeypatch): + _preimport_real_module("afd_plugin.v1.worker.npu.mla_graph") fake_ascend = ModuleType("vllm_ascend") fake_ascend.__path__ = [] fake_compilation = ModuleType("vllm_ascend.compilation") @@ -69,6 +87,8 @@ def _graph_params( def _load_forward_context_module(monkeypatch): + _preimport_real_module("afd_plugin.v1.worker.ubatch_wrapper") + _preimport_real_module("afd_plugin.v1.worker.npu.forward_context") fake_vllm = ModuleType("vllm") fake_vllm.__path__ = [] fake_config = ModuleType("vllm.config") @@ -139,6 +159,7 @@ def __init__(self, **kwargs): def _load_ubatch_wrapper_module(monkeypatch): + _preimport_real_module("afd_plugin.v1.worker.ubatch_wrapper") class FakeStream: def __init__(self, device=None): self.device = device diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index 1ff670e8..4689d487 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -410,6 +410,16 @@ def _new_ffn_runner(): runner.prof = None runner.device = SimpleNamespace(type="npu") runner._is_shutdown = False + # _ffn_layer_indices reads these; production runs with the gate on the + # Attention side, so every layer index is a MoE FFN layer here. + runner.afd_config = SimpleNamespace(compute_gate_on_attention=True) + runner.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + n_routed_experts=8, + first_k_dense_replace=0, + moe_layer_freq=1, + ), + ) return runner @@ -417,7 +427,9 @@ def _new_ffn_worker(): _require_npu_runtime() from afd_plugin.v1.worker.npu.ffn_worker import AFDNPUFFNWorker - return object.__new__(AFDNPUFFNWorker) + worker = object.__new__(AFDNPUFFNWorker) + worker._ffn_loop_error = None + return worker @pytest.mark.parametrize( @@ -517,6 +529,7 @@ def __init__(self, *args, **kwargs): def test_npu_attention_runner_builds_and_sets_metadata(): + torch = pytest.importorskip("torch") runner = _new_attention_runner() runner.vllm_config = _vllm_config(role="attention") runner.connector = _RecordingConnector() @@ -524,9 +537,10 @@ def test_npu_attention_runner_builds_and_sets_metadata(): runner._afd_is_graph_capturing = False runner._afd_pending_metadata = None runner._afd_transaction_counter = 0 + runner._afd_suppress_metadata_send = False forward_context = SimpleNamespace( additional_kwargs={}, - dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=[1]), + dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor([1])), ubatch_slices=None, batch_descriptor=SimpleNamespace(num_tokens=5), ) @@ -583,6 +597,7 @@ def test_npu_attention_runner_builds_dp_fallback(): def test_npu_attention_runner_sends_graph_flags(): + torch = pytest.importorskip("torch") runner = _new_attention_runner() runner.vllm_config = _vllm_config(role="attention") runner.connector = _RecordingConnector() @@ -591,7 +606,7 @@ def test_npu_attention_runner_sends_graph_flags(): runner._afd_transaction_counter = 0 runner._afd_pending_metadata = runner._build_afd_metadata(None, 3) - runner._send_dp_metadata(SimpleNamespace(num_tokens_across_dp_cpu=[3]), None) + runner._send_dp_metadata(SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor([3])), None) assert runner.connector.dp_metadata_updates[0][1:] == (True, True) assert runner.connector.sent_dp_metadata_lists[0][1:] == (True, True) @@ -683,13 +698,12 @@ def send_dp_metadata(dp_metadata, ubatch_slices): runner._send_dp_metadata = send_dp_metadata desc = SimpleNamespace(num_tokens=12, uniform=True, num_active_loras=0) - result = runner._warmup_and_capture( + runner._warmup_and_capture( desc, CUDAGraphMode.FULL, allow_microbatching=True, ) - assert result is True assert [call[1]["allow_microbatching"] for call in dummy_calls] == [ False, False, @@ -1053,7 +1067,9 @@ def make_attn_group(group_id): runner.attn_groups = [[make_attn_group(0), make_attn_group(1)]] runner.max_model_len = 105 runner.optimistic_seq_lens_cpu = torch.tensor([105], dtype=torch.int32) - runner.use_dcp = False + # vLLM-Ascend v0.26 exposes use_dcp as a read-only property derived from + # dcp_size, so the fixture sets the underlying size instead. + runner.dcp_size = 1 runner.use_async_spec_decode = False runner.input_batch = SimpleNamespace( block_table=[block_table], @@ -1215,7 +1231,7 @@ def create_metadata_builders( lambda self, *args, **kwargs: initialized, ) - assert runner.initialize_attn_backend() is initialized + runner.initialize_attn_backend(None) assert create_calls == [3] assert len(attn_group.metadata_builders) == 3 From d6de6754268d7cfad0ad8d765c77cb599d4b71bb Mon Sep 17 00:00:00 2001 From: ShwStone Date: Thu, 13 Aug 2026 16:16:58 +0800 Subject: [PATCH 2/3] test(npu): re-add conftest.py vllm_ascend.ops preimport Restore the circular-import workaround that was dropped when the old conftest.py (with now-obsolete E2E fixtures) was deleted on main. Without it, pytest collection trips vllm_ascend's device_op -> ops -> experts_selector cycle (ImportError: cannot import name 'DeviceOperator'). Co-Authored-By: Claude --- tests/conftest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index bd4fe8f2..c5249df1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,17 @@ # Covers the roughly 64-second nested lm-eval/vLLM cleanup bound with buffer. RUNNER_CLEANUP_TIMEOUT_S = 90 +# vLLM-Ascend 80d8c194f (v0.26 baseline) has an inherent circular import: +# device/device_op.py -> ops/__init__ -> fused_moe -> experts_selector.py -> +# back to the partially initialized device_op. Real ``vllm serve`` imports +# ``vllm_ascend.ops`` first and never trips it, but pytest's import order does. +# Pre-importing ``vllm_ascend.ops`` here breaks the cycle for the test run. +# Environments without vllm_ascend (CPU-only CI) simply skip this. +try: # pragma: no cover - environment-dependent import ordering fix + import vllm_ascend.ops # noqa: F401 +except Exception: + pass + def run_runner(command: list[str], env: dict[str, str] | None = None) -> None: """Run an E2E runner command and forward cancellation to the process group.""" From fe59c1ecffe7706caed22a489e34c60f09c8cf8a Mon Sep 17 00:00:00 2001 From: ShwStone Date: Fri, 14 Aug 2026 11:13:11 +0800 Subject: [PATCH 3/3] test(e2e): add afd-async-ubatch scenario (A2F1, CAM async eager) A2F1 topology (2 attention + 1 FFN) with CAMAsyncAFDConnector, async=true, compute_gate_on_attention=true, eager execution, and gpu-memory-utilization 0.8 (CAM dispatch allocates an ~8.6 GB HCCL pool outside the torch pool; the default 0.92 fails with EL0004). Ubatching is not enabled yet. Known issue documented for handoff: with DP2 attention, a cold-start single request deadlocks in async_dispatch_send because the CAM dispatch collective requires all attention ranks while vLLM DP only schedules engines that have requests. A second request (routed to the idle engine) unlocks it. Co-Authored-By: Claude --- tests/e2e/runner.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index 6732c21e..371e472a 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -31,6 +31,7 @@ ASYNC_CAM_ATTENTION_RANKS = 2 ASYNC_CAM_FFN_RANKS = 2 ASYNC_CAM_ATTENTION_TP_SIZE = 2 +ASYNC_UBATCH_SCENARIO = "afd-async-ubatch" PROCESS_TERMINATION_TIMEOUT_S = 20 PROCESS_POLL_INTERVAL_S = 0.2 PROCESS_REAP_TIMEOUT_S = 5 @@ -193,6 +194,7 @@ def parse_args() -> argparse.Namespace: "afd-graph-dbo", "afd-graph-dbo-2a2f", ASYNC_CAM_SCENARIO, + ASYNC_UBATCH_SCENARIO, ], required=True, help="Fixed E2E scenario to run.", @@ -294,6 +296,7 @@ def parse_args() -> argparse.Namespace: def configure_scenario(args: argparse.Namespace) -> None: """Set topology and features for the selected fixed scenario.""" is_async_cam = args.scenario == ASYNC_CAM_SCENARIO + is_async_ubatch = args.scenario == ASYNC_UBATCH_SCENARIO scenario_settings = { "baseline-graph": (True, True, False, 1, 0), "afd-eager": (False, False, False, 2, 1), @@ -307,6 +310,7 @@ def configure_scenario(args: argparse.Namespace) -> None: ASYNC_CAM_ATTENTION_RANKS, ASYNC_CAM_FFN_RANKS, ), + ASYNC_UBATCH_SCENARIO: (False, False, False, 2, 1), } baseline, use_graph, enable_dbo, attention_ranks, ffn_ranks = scenario_settings[ args.scenario @@ -332,6 +336,19 @@ def configure_scenario(args: argparse.Namespace) -> None: args.afd_connector_extra_config = [ json.dumps(extra_config, separators=(",", ":")), ] + if is_async_ubatch: + args.afd_connector = ASYNC_AFD_CONNECTOR + args.afd_async = True + args.compute_gate_on_attention = True + # CAM dispatch allocates its HCCL communication pool (~8.6 GB) outside + # the torch memory pool; the default 0.92 GPU memory utilization leaves + # too little room and fails with EL0004. 0.8 leaves enough headroom. + if not any( + arg == "--gpu-memory-utilization" + or arg.startswith("--gpu-memory-utilization=") + for arg in args.common_vllm_arg + ): + args.common_vllm_arg.extend(["--gpu-memory-utilization", "0.8"]) if use_graph: args.cudagraph_capture_size = 8 if enable_dbo: