diff --git a/afd_plugin/connectors/npu/async_cam.py b/afd_plugin/connectors/npu/async_cam.py index 00dd128a..a93e3085 100644 --- a/afd_plugin/connectors/npu/async_cam.py +++ b/afd_plugin/connectors/npu/async_cam.py @@ -56,7 +56,13 @@ AFDTransferMetadata, AFDTransferState, ) -from afd_plugin.distributed import init_afd_process_group +from afd_plugin.distributed import ( + create_hccl_process_group_options, + init_afd_process_group, +) +from afd_plugin.distributed.cam_hccl_buffer import ( + derive_cam_hccl_buffer_plan_from_config, +) if TYPE_CHECKING: from torch.distributed.distributed_c10d import ProcessGroup @@ -238,8 +244,8 @@ def __init__( Communication resources are created collectively by ``init_afd_connector``. ``role_rank`` is resolved before connector - construction; ``attn_ranks_per_dp`` is used as the CAM Attention TP - width. + construction; ``attn_ranks_per_dp`` supplies the number of NPUs in one + Attention data-parallel group. """ super().__init__(rank, local_rank, vllm_config, afd_config, role_rank) self._initialized = False @@ -251,7 +257,21 @@ def __init__( self.group_name = "" self.max_seq_len = vllm_config.scheduler_config.max_num_batched_tokens self.comm_id = CAM_COMM_ID - self.tp_size = self.extra_info.attn_ranks_per_dp + self.num_npus_per_dp_group = self.extra_info.attn_ranks_per_dp + self.hccl_buffer_plan = derive_cam_hccl_buffer_plan_from_config( + vllm_config, + afd_config, + ) + self.hccl_buffer_size_mb = self.hccl_buffer_plan.buffer_size_mb_for_role( + afd_config.role, + ) + logger.info( + "CAM async %s HCCL buffer size is %d MB (auto-derived with " + "1.1x headroom from %d required bytes)", + afd_config.role, + self.hccl_buffer_size_mb, + self.hccl_buffer_plan.required_bytes_for_role(afd_config.role), + ) self.cam_pg: ProcessGroup | None = None self.topology = build_async_topology( afd_config, @@ -292,6 +312,9 @@ def init_afd_connector(self) -> None: rank=self.world_rank, group_name=AFD_ASYNC_CAM_GROUP_NAME, timeout=timedelta(minutes=30), + pg_options=create_hccl_process_group_options( + self.hccl_buffer_size_mb, + ), ) backend = self.cam_pg._get_backend(torch.device("npu")) self.group_name = str(backend.get_hccl_comm_name(self.world_rank)) @@ -523,7 +546,7 @@ def send_attn_output( rank=self.world_rank, world_size=self.topology.world_size, layer_idx=states.layer_idx, - tp_size=self.tp_size, + num_npus_per_dp_group=self.num_npus_per_dp_group, dynamic_quant=self.dynamic_quant, group_name=self.group_name, ) @@ -542,7 +565,7 @@ def send_attn_output( self.world_rank, self.topology.world_size, states.layer_idx, - self.tp_size, + self.num_npus_per_dp_group, self.dynamic_quant, self.group_name, ) @@ -691,7 +714,7 @@ def recv_attn_output( expert_per_rank=self.expert_per_rank, rank=self.world_rank, world_size=self.topology.world_size, - tp_size=self.tp_size, + num_npus_per_dp_group=self.num_npus_per_dp_group, dynamic_quant=self.dynamic_quant, group_name=self.group_name, ) @@ -707,7 +730,7 @@ def recv_attn_output( self.expert_per_rank, self.world_rank, self.topology.world_size, - self.tp_size, + self.num_npus_per_dp_group, self.dynamic_quant, self.group_name, ) @@ -782,7 +805,7 @@ def send_ffn_output( expert_per_rank=self.expert_per_rank, rank=self.world_rank, world_size=self.topology.world_size, - tp_size=self.tp_size, + num_npus_per_dp_group=self.num_npus_per_dp_group, group_name=self.group_name, ) torch.ops.umdk_cam_op_lib.async_combine_send( @@ -799,7 +822,7 @@ def send_ffn_output( self.expert_per_rank, self.world_rank, self.topology.world_size, - self.tp_size, + self.num_npus_per_dp_group, self.group_name, ) diff --git a/afd_plugin/connectors/npu/camp2p.py b/afd_plugin/connectors/npu/camp2p.py index 32f25424..429e45aa 100644 --- a/afd_plugin/connectors/npu/camp2p.py +++ b/afd_plugin/connectors/npu/camp2p.py @@ -50,7 +50,14 @@ recv_control_payload, send_control_payload, ) -from afd_plugin.distributed import init_afd_process_group, topology_from_config +from afd_plugin.distributed import ( + create_hccl_process_group_options, + init_afd_process_group, + topology_from_config, +) +from afd_plugin.distributed.cam_hccl_buffer import ( + derive_cam_hccl_buffer_plan_from_config, +) if TYPE_CHECKING: from vllm.config import VllmConfig @@ -283,6 +290,20 @@ def __init__( self.hidden_size = hf_config.hidden_size self.num_experts_per_tok = hf_config.num_experts_per_tok self.num_routed_experts = hf_config.n_routed_experts + self.hccl_buffer_plan = derive_cam_hccl_buffer_plan_from_config( + vllm_config, + afd_config, + ) + self.hccl_buffer_size_mb = self.hccl_buffer_plan.buffer_size_mb_for_role( + afd_config.role, + ) + logger.info( + "CAM P2P %s HCCL buffer size is %d MB (auto-derived with " + "1.1x headroom from %d required bytes)", + afd_config.role, + self.hccl_buffer_size_mb, + self.hccl_buffer_plan.required_bytes_for_role(afd_config.role), + ) self.control_plane = CAMP2pAFDControlPlane(self) @property @@ -325,6 +346,9 @@ def init_afd_connector(self) -> None: rank=self.world_rank, group_name=group_name, timeout=timedelta(minutes=30), + pg_options=create_hccl_process_group_options( + self.hccl_buffer_size_mb, + ), ) self.afd_pg_list.append(afd_pg) backend = afd_pg._get_backend(torch.device("npu")) @@ -346,6 +370,9 @@ def init_afd_connector(self) -> None: rank=self.world_rank, group_name="afd_moe", timeout=timedelta(minutes=30), + pg_options=create_hccl_process_group_options( + self.hccl_buffer_size_mb, + ), ) backend = self.ffn_pg._get_backend(torch.device("npu")) self.hccl_comm_name1 = str( diff --git a/afd_plugin/distributed/__init__.py b/afd_plugin/distributed/__init__.py index eefb44ea..8769568e 100644 --- a/afd_plugin/distributed/__init__.py +++ b/afd_plugin/distributed/__init__.py @@ -12,7 +12,11 @@ def __getattr__(name: str): - if name in {"DefaultProcessGroupSwitcher", "init_afd_process_group"}: + if name in { + "DefaultProcessGroupSwitcher", + "create_hccl_process_group_options", + "init_afd_process_group", + }: from afd_plugin.distributed import afd_process_group value = getattr(afd_process_group, name) @@ -25,6 +29,7 @@ def __getattr__(name: str): "AFDRankMapping", "DefaultProcessGroupSwitcher", "build_rank_mapping", + "create_hccl_process_group_options", "init_afd_process_group", "resolve_role_rank", "topology_from_config", diff --git a/afd_plugin/distributed/afd_process_group.py b/afd_plugin/distributed/afd_process_group.py index 24102973..5cb2cb6d 100644 --- a/afd_plugin/distributed/afd_process_group.py +++ b/afd_plugin/distributed/afd_process_group.py @@ -39,6 +39,19 @@ def __exit__(self, exc_type: object, exc_value: object, tb: object) -> None: _update_default_pg(self.default_group) +def create_hccl_process_group_options(hccl_buffer_size: int) -> Any: + """Create fresh HCCL options for one plugin-owned process group. + + ``hccl_buffer_size`` is expressed in MB. Per-process-group options keep CAM + buffer sizing independent from process-wide ``HCCL_BUFFSIZE``. + """ + import torch_npu + + options = torch_npu._C._distributed_c10d.ProcessGroupHCCL.Options() + options.hccl_config = {"hccl_buffer_size": hccl_buffer_size} + return options + + def init_afd_process_group( *, backend: str, @@ -99,5 +112,6 @@ def init_afd_process_group( __all__ = [ "DefaultProcessGroupSwitcher", + "create_hccl_process_group_options", "init_afd_process_group", ] diff --git a/afd_plugin/distributed/cam_hccl_buffer.py b/afd_plugin/distributed/cam_hccl_buffer.py new file mode 100644 index 00000000..3aeb9fcb --- /dev/null +++ b/afd_plugin/distributed/cam_hccl_buffer.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CAM HCCL buffer sizing and memory-headroom warnings.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vllm.config import VllmConfig + + from afd_plugin.config import AFDConfig + +CAM_ATTENTION_ELEMENT_SIZE_BYTES = 2 +CAM_DYNAMIC_QUANT_MOE_TOKEN_SIZE_BYTES = 6176 +CAM_NON_QUANT_MOE_TOKEN_SIZE_BYTES = 12288 +CAM_BUFFER_SAFETY_FACTOR_NUMERATOR = 11 +CAM_BUFFER_SAFETY_FACTOR_DENOMINATOR = 10 +CAM_MEMORY_RESERVE_FACTOR_NUMERATOR = 5 +CAM_MEMORY_RESERVE_FACTOR_DENOMINATOR = 2 +MEBIBYTE = 1024**2 + +logger = logging.getLogger(__name__) + + +def _ceil_div(dividend: int, divisor: int) -> int: + return (dividend + divisor - 1) // divisor + + +@dataclass(frozen=True, slots=True) +class CAMHCCLBufferPlan: + """Derived role-local HCCL buffer requirements.""" + + attention_required_bytes: int + moe_required_bytes: int + attention_buffer_size_mb: int + ffn_buffer_size_mb: int + + def buffer_size_mb_for_role(self, role: str) -> int: + """Return the independently derived HCCL setting for one AFD role.""" + if role == "attention": + return self.attention_buffer_size_mb + if role == "ffn": + return self.ffn_buffer_size_mb + raise ValueError(f"unsupported AFD role for CAM buffer sizing: {role!r}") + + def required_bytes_for_role(self, role: str) -> int: + """Return the pre-headroom byte requirement for one AFD role.""" + if role == "attention": + return self.attention_required_bytes + if role == "ffn": + return self.moe_required_bytes + raise ValueError(f"unsupported AFD role for CAM buffer sizing: {role!r}") + + +def derive_cam_hccl_buffer_plan( + *, + hidden_size: int, + max_batch_tokens: int, + num_npus_per_dp_group: int, + topk: int, + attention_rank_size: int, + dynamic_quant: int, +) -> CAMHCCLBufferPlan: + """Derive independent Attention and FFN HCCL buffers with 10% headroom. + + The Attention side includes routed and shared-expert payloads, represented + by ``topk + 1``. The MoE side uses the CAM-provided per-token byte width and + intentionally has no ``topk + 1`` multiplier. + """ + if dynamic_quant not in (0, 1): + raise ValueError(f"dynamic_quant must be 0 or 1, got {dynamic_quant}") + if num_npus_per_dp_group <= 0: + raise ValueError( + f"num_npus_per_dp_group must be positive, got {num_npus_per_dp_group}", + ) + + tokens_per_npu = _ceil_div(max_batch_tokens, num_npus_per_dp_group) + attention_required_bytes = ( + CAM_ATTENTION_ELEMENT_SIZE_BYTES * hidden_size * tokens_per_npu * (topk + 1) + ) + moe_token_size_bytes = ( + CAM_DYNAMIC_QUANT_MOE_TOKEN_SIZE_BYTES + if dynamic_quant + else CAM_NON_QUANT_MOE_TOKEN_SIZE_BYTES + ) + moe_required_bytes = attention_rank_size * moe_token_size_bytes * tokens_per_npu + attention_buffered_bytes = _ceil_div( + attention_required_bytes * CAM_BUFFER_SAFETY_FACTOR_NUMERATOR, + CAM_BUFFER_SAFETY_FACTOR_DENOMINATOR, + ) + moe_buffered_bytes = _ceil_div( + moe_required_bytes * CAM_BUFFER_SAFETY_FACTOR_NUMERATOR, + CAM_BUFFER_SAFETY_FACTOR_DENOMINATOR, + ) + return CAMHCCLBufferPlan( + attention_required_bytes=attention_required_bytes, + moe_required_bytes=moe_required_bytes, + attention_buffer_size_mb=_ceil_div( + attention_buffered_bytes, + MEBIBYTE, + ), + ffn_buffer_size_mb=_ceil_div(moe_buffered_bytes, MEBIBYTE), + ) + + +def derive_cam_hccl_buffer_plan_from_config( + vllm_config: VllmConfig, + afd_config: AFDConfig, +) -> CAMHCCLBufferPlan: + """Derive CAM buffer sizes for either Ascend CAM connector.""" + from afd_plugin.config import ( + AFD_ASYNC_CONNECTOR, + connector_extra_config_from_source, + ) + from afd_plugin.config_utils import ( + coerce_extra_int, + coerce_extra_positive_int, + ) + + extra_config = connector_extra_config_from_source(vllm_config) + if afd_config.connector == AFD_ASYNC_CONNECTOR: + num_npus_per_dp_group = coerce_extra_positive_int( + extra_config.get("attn_ranks_per_dp", 1), + field_name="attn_ranks_per_dp", + ) + dynamic_quant = coerce_extra_int( + extra_config.get("dynamicQuant", 0), + field_name="dynamicQuant", + ) + elif afd_config.connector == "CAMP2pAFDConnector": + # CAMP2P currently supports TP as the only intra-DP NPU dimension. + num_npus_per_dp_group = int( + vllm_config.parallel_config.tensor_parallel_size, + ) + dynamic_quant = 0 + else: + raise ValueError( + "CAM HCCL buffer sizing requires CAMAsyncAFDConnector or " + f"CAMP2pAFDConnector, got {afd_config.connector!r}", + ) + + hf_config = vllm_config.model_config.hf_config + return derive_cam_hccl_buffer_plan( + hidden_size=hf_config.hidden_size, + max_batch_tokens=vllm_config.scheduler_config.max_num_batched_tokens, + num_npus_per_dp_group=num_npus_per_dp_group, + topk=hf_config.num_experts_per_tok, + attention_rank_size=afd_config.num_attention_ranks, + dynamic_quant=dynamic_quant, + ) + + +def warn_if_cam_memory_headroom_is_low( + vllm_config: VllmConfig, + afd_config: AFDConfig, + total_device_memory_bytes: int, +) -> None: + """Warn when configured utilization leaves less than 2.5 CAM buffers.""" + from afd_plugin.config import AFD_ASYNC_CONNECTOR + + if afd_config.connector not in (AFD_ASYNC_CONNECTOR, "CAMP2pAFDConnector"): + return + + buffer_plan = derive_cam_hccl_buffer_plan_from_config(vllm_config, afd_config) + buffer_size_mb = buffer_plan.buffer_size_mb_for_role(afd_config.role) + buffer_size_bytes = buffer_size_mb * MEBIBYTE + required_reserve_bytes = _ceil_div( + buffer_size_bytes * CAM_MEMORY_RESERVE_FACTOR_NUMERATOR, + CAM_MEMORY_RESERVE_FACTOR_DENOMINATOR, + ) + gpu_memory_utilization = vllm_config.cache_config.gpu_memory_utilization + configured_memory_bytes = int(total_device_memory_bytes * gpu_memory_utilization) + available_reserve_bytes = max( + 0, + total_device_memory_bytes - configured_memory_bytes, + ) + if available_reserve_bytes >= required_reserve_bytes: + return + + recommended_maximum_utilization = max( + 0.0, + (total_device_memory_bytes - required_reserve_bytes) + / total_device_memory_bytes, + ) + logger.warning( + "CAM %s %s rank has %d bytes outside gpu_memory_utilization, below " + "the recommended %d bytes (2.5x its %d MB HCCL buffer); consider " + "setting gpu_memory_utilization to %.6f or lower. The configured " + "value %.6f is unchanged.", + afd_config.connector, + afd_config.role, + available_reserve_bytes, + required_reserve_bytes, + buffer_size_mb, + recommended_maximum_utilization, + gpu_memory_utilization, + ) + + +__all__ = [ + "CAMHCCLBufferPlan", + "derive_cam_hccl_buffer_plan", + "derive_cam_hccl_buffer_plan_from_config", + "warn_if_cam_memory_headroom_is_low", +] diff --git a/afd_plugin/v1/worker/npu/attention_worker.py b/afd_plugin/v1/worker/npu/attention_worker.py index d6a24861..5b9ea3e3 100644 --- a/afd_plugin/v1/worker/npu/attention_worker.py +++ b/afd_plugin/v1/worker/npu/attention_worker.py @@ -15,6 +15,10 @@ fix_all2all_backend_for_afd, npu_afd_num_ubatches, ) +from afd_plugin.config import parse_afd_config +from afd_plugin.distributed.cam_hccl_buffer import ( + warn_if_cam_memory_headroom_is_low, +) from afd_plugin.model_executor.models.model_utils import get_afd_model_config from afd_plugin.v1.worker.npu.attention_model_runner import ( AFDNPUAttentionModelRunner, @@ -49,6 +53,15 @@ def init_device(self) -> None: ) self.device = self._init_device() + afd_config = parse_afd_config( + self.vllm_config, + expected_role="attention", + ) + warn_if_cam_memory_headroom_is_low( + self.vllm_config, + afd_config, + int(self.init_snapshot.total_memory), + ) init_workspace_manager( self.device, npu_afd_num_ubatches(self.vllm_config), diff --git a/afd_plugin/v1/worker/npu/ffn_worker.py b/afd_plugin/v1/worker/npu/ffn_worker.py index 54b8120a..1a0244ae 100644 --- a/afd_plugin/v1/worker/npu/ffn_worker.py +++ b/afd_plugin/v1/worker/npu/ffn_worker.py @@ -19,6 +19,10 @@ fix_all2all_backend_for_afd, npu_afd_num_ubatches, ) +from afd_plugin.config import parse_afd_config +from afd_plugin.distributed.cam_hccl_buffer import ( + warn_if_cam_memory_headroom_is_low, +) from afd_plugin.model_executor.models.model_utils import get_afd_model_config from afd_plugin.v1.worker.npu.ffn_model_runner import AFDNPUFFNModelRunner from afd_plugin.validation import NPU_FFN_WORKER_FQCN, assert_compatible_afd_stack @@ -63,6 +67,15 @@ def init_device(self) -> None: raise RuntimeError("AFD NPU FFN supports only vllm-ascend MRv1") self.device = self._init_device() + afd_config = parse_afd_config( + self.vllm_config, + expected_role="ffn", + ) + warn_if_cam_memory_headroom_is_low( + self.vllm_config, + afd_config, + int(self.init_snapshot.total_memory), + ) init_workspace_manager( self.device, npu_afd_num_ubatches(self.vllm_config), diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index 01d1b876..f5171b58 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -162,7 +162,7 @@ spelling used by the recipes. | Field | Type | Default | Meaning and constraint | | --- | --- | --- | --- | | `dynamicQuant` | `int` | `0` | Enables CAM dispatch/combine dynamic-quant metadata. Only `0` and `1` are accepted. With `1`, FFN receives quantized routed activations plus scale tensors and must return output compatible with combine-send. | -| `attn_ranks_per_dp` | `int` | `1` | Positive Attention TP rank count per DP replica. It is the CAM Attention grouping width and is independent of the FFN process's local TP size. | +| `attn_ranks_per_dp` | `int` | `1` | Positive number of Attention NPUs in each DP group. The current runtime supports only TP within this group, so this value must equal Attention TP size. It is independent of the FFN process's local TP size. | | `async_moe_ubatching` | `bool` | `false` | Enables AFD-managed asynchronous MoE-only ubatching. | | `async_moe_num_ubatches` | `int` | `2` | Number of asynchronous MoE stages. Only `2` is supported. | | `async_moe_split` | `str` | `"request"` | `"request"` requires two scheduled requests and preserves their boundaries. `"token"` balances flattened real tokens and requires Attention TP greater than one. Both modes reject context parallelism; FFN may independently use TP1. | @@ -173,6 +173,38 @@ For a DP+SP deployment such as DP3TP2 Attention + DP2TP1/EP2 FFN, set DP+TP Attention, leave FlashComm1 disabled on both roles. In either case, FFN consumes expert-routed CAM work items and may independently use TP1. +### Automatic HCCL buffer sizing + +The plugin derives the CAM HCCL process-group buffer at startup for both CAM +Async and synchronous CAMP2P. For CAM Async, the setting is applied only to the +`afd_async_cam` process group, independently of the global `HCCL_BUFFSIZE` +environment variable. Attention and FFN ranks use their own role-specific +values: + +```text +attention_bytes = 2 * hidden_size * ceil(max_num_batched_tokens / attn_ranks_per_dp) * (topk + 1) +ffn_bytes = num_attention_ranks * (6176 if dynamicQuant else 12288) * ceil(max_num_batched_tokens / attn_ranks_per_dp) +role_buffer_mb = ceil(1.1 * role_bytes / 1_MiB) +``` + +The MoE/FFN formula intentionally has no `topk + 1` multiplier. Every rank logs +its selected role and buffer size at INFO level. This role-local configuration +must be validated with the target torch-npu/HCCL stack because Attention and +FFN ranks participate in the same CAM world with different buffer settings. + +Before model and KV-cache allocation, the worker checks whether the memory +outside `gpu_memory_utilization` is at least 2.5 times its role-local buffer. If +the configured utilization leaves less headroom, the worker emits a warning +with a recommended maximum utilization. It does not modify the configured +value. + +The plugin does not modify or unset `HCCL_BUFFSIZE`. The CAM process group uses +its independently derived per-group value, so `HCCL_BUFFSIZE` does not size the +CAM buffer. If you set the environment variable, it remains available to +vLLM-Ascend and HCCL for other process groups such as TP, DP, PCP, or EP. Leave +it unset if those process groups should use their normal HCCL defaults; set it +only when you intentionally want to tune them. + ## Native DBO and async MoE ubatching are different ### vLLM native DBO @@ -274,7 +306,6 @@ the essential setup is: export ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM:${ASCEND_CUSTOM_OPP_PATH} export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH} export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.1/opp/vendors/CAM/op_api:${LD_LIBRARY_PATH} -export HCCL_BUFFSIZE=4096 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True ``` diff --git a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md index 86326154..059b6168 100644 --- a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md @@ -47,6 +47,29 @@ The connector creates these communication groups: - one Gloo group used to send DP metadata from participating Attention ranks to FFN ranks. +## Automatic HCCL buffer sizing + +The plugin derives independent Attention and FFN HCCL buffer sizes at startup +and applies the role-local value to every CAMP2P-owned HCCL group. The metadata +group uses Gloo and is unaffected. The current CAMP2P runtime supports only TP +within a DP group, so the number of NPUs per DP group is its TP size: + +```text +attention_bytes = 2 * hidden_size * ceil(max_num_batched_tokens / num_npus_per_dp_group) * (topk + 1) +ffn_bytes = num_attention_ranks * 12288 * ceil(max_num_batched_tokens / num_npus_per_dp_group) +role_buffer_mb = ceil(1.1 * role_bytes / 1_MiB) +``` + +The FFN formula intentionally has no `topk + 1` multiplier. Every rank logs its +selected role and buffer size at INFO level. If the memory outside +`gpu_memory_utilization` is less than 2.5 times the role-local buffer, the +worker warns and recommends a maximum utilization without changing the +configured value. + +The process-group setting is independent of the global `HCCL_BUFFSIZE` +environment variable. The plugin neither modifies nor unsets that variable, +so it remains available for other HCCL groups. + ## DBO and ubatching DBO is configured with vLLM CLI flags, not inside the `afd` object: diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md index a9f65846..e9ae52ea 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md @@ -251,6 +251,10 @@ vllm serve /path/to/DeepSeek-V3.2 \ - FFN side: `EP8`. - Connector: `CAMAsyncAFDConnector`. - Current scope: PD-disaggregated prefill stage only. +- `HCCL_BUFFSIZE` is optional and is not used to size the CAM process-group + buffer. If set, it is preserved for other HCCL process groups; CAM uses its + independently auto-derived Attention/FFN value. Leave it unset unless those + other groups need explicit tuning.
Node0 Attention Deployment Command (DP0-DP1) @@ -267,7 +271,6 @@ export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} -export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export AFD_FORCE_BALANCED_TOPK_IDS=1 @@ -331,7 +334,6 @@ export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} -export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export AFD_FORCE_BALANCED_TOPK_IDS=1 @@ -396,7 +398,6 @@ export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export HCCL_OP_EXPANSION_MODE=AIV export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/opp/vendors/CAM/op_api/lib:${LD_LIBRARY_PATH:-} -export HCCL_BUFFSIZE=4096 export VLLM_ASCEND_ENABLE_CONTEXT_PARALLEL=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export AFD_FORCE_BALANCED_TOPK_IDS=1 diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/README.md index e585cda7..e8f0a252 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/README.md @@ -13,6 +13,10 @@ Both launch scripts set `--gpu-memory-utilization 0.8`. The remaining device memory is intentional headroom for CAM communication buffers and runtime allocations on this multi-node topology. +The scripts do not set `HCCL_BUFFSIZE`; CAM uses its independently derived +Attention/FFN process-group buffer. An inherited `HCCL_BUFFSIZE` remains +available for intentionally tuning other HCCL process groups. + The launch scripts intentionally do not use SSH or manage the other node's processes. Start each role through the cluster job system so teardown remains owned by that system. diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/attention_dp2tp8.sh b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/attention_dp2tp8.sh index 30e23163..012eee82 100755 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/attention_dp2tp8.sh +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/attention_dp2tp8.sh @@ -16,7 +16,6 @@ VISIBLE_DEVICES="${ASCEND_RT_VISIBLE_DEVICES:-$DEFAULT_VISIBLE_DEVICES}" export ASCEND_RT_VISIBLE_DEVICES="$VISIBLE_DEVICES" export AFD_FORCE_BALANCED_TOPK_IDS=0 -export HCCL_BUFFSIZE="${HCCL_BUFFSIZE:-4096}" export HCCL_OP_EXPANSION_MODE="${HCCL_OP_EXPANSION_MODE:-AIV}" export OMP_PROC_BIND="${OMP_PROC_BIND:-false}" export OMP_NUM_THREADS="${OMP_NUM_THREADS:-10}" diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/ffn_ep16.sh b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/ffn_ep16.sh index 7579ab1a..26385e94 100755 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/ffn_ep16.sh +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/ffn_ep16.sh @@ -16,7 +16,6 @@ VISIBLE_DEVICES="${ASCEND_RT_VISIBLE_DEVICES:-$DEFAULT_VISIBLE_DEVICES}" export ASCEND_RT_VISIBLE_DEVICES="$VISIBLE_DEVICES" export AFD_FORCE_BALANCED_TOPK_IDS=0 -export HCCL_BUFFSIZE="${HCCL_BUFFSIZE:-4096}" export HCCL_OP_EXPANSION_MODE="${HCCL_OP_EXPANSION_MODE:-AIV}" export OMP_PROC_BIND="${OMP_PROC_BIND:-false}" export OMP_NUM_THREADS="${OMP_NUM_THREADS:-10}" diff --git a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md index f7c57522..80f026fa 100644 --- a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md +++ b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/README.md @@ -138,6 +138,12 @@ AFD async-DP. The `--async-scheduling` CLI option used by all cases is a vLLM scheduler optimization and is unrelated to AFD async-DP or `CAMAsyncAFDConnector`. +The AFD launchers do not set `HCCL_BUFFSIZE`. CAMP2P uses independently +auto-derived Attention and FFN process-group buffers. An inherited +`HCCL_BUFFSIZE` remains available for intentionally tuning other HCCL process +groups. The EP64 baseline keeps its existing global setting because it does +not use a CAM connector. + AFD attention and FFN workers also enable Dual Batch Overlap (DBO): ```text diff --git a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_attention.sh b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_attention.sh index 694aee0f..e83ab0b7 100644 --- a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_attention.sh +++ b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_attention.sh @@ -50,7 +50,6 @@ if [[ "$INPUT_LENGTH" == 16384 && "$BATCH_SIZE" != 28 ]] || fi export ASCEND_RT_VISIBLE_DEVICES="$VISIBLE_DEVICES" -export HCCL_BUFFSIZE="${HCCL_BUFFSIZE:-512}" export HCCL_OP_EXPANSION_MODE="${HCCL_OP_EXPANSION_MODE:-AIV}" export OMP_PROC_BIND="${OMP_PROC_BIND:-false}" export OMP_NUM_THREADS="${OMP_NUM_THREADS:-10}" diff --git a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_ffn.sh b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_ffn.sh index 34177c0f..388c6efc 100644 --- a/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_ffn.sh +++ b/recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/afd_ffn.sh @@ -45,7 +45,6 @@ fi FFN_BATCH_SIZE=$((ATTENTION_RANKS * BATCH_SIZE / FFN_RANKS)) export ASCEND_RT_VISIBLE_DEVICES="$VISIBLE_DEVICES" -export HCCL_BUFFSIZE="${HCCL_BUFFSIZE:-2048}" export HCCL_OP_EXPANSION_MODE="${HCCL_OP_EXPANSION_MODE:-AIV}" export OMP_PROC_BIND="${OMP_PROC_BIND:-false}" export OMP_NUM_THREADS="${OMP_NUM_THREADS:-10}" diff --git a/tests/unit/connectors/test_async_cam_connector.py b/tests/unit/connectors/test_async_cam_connector.py index 8e4102a2..697f8985 100644 --- a/tests/unit/connectors/test_async_cam_connector.py +++ b/tests/unit/connectors/test_async_cam_connector.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import sys from types import ModuleType, SimpleNamespace @@ -29,6 +30,9 @@ CAMAsyncAFDConnector, build_async_topology, ) +from afd_plugin.distributed.afd_process_group import ( # noqa: E402 + create_hccl_process_group_options, +) class _FakeScalar: @@ -186,6 +190,42 @@ def test_async_extra_info_parses_async_moe_fields(): assert extra_info.async_moe_split == "request" +def test_create_hccl_process_group_options_sets_per_group_buffer(monkeypatch): + class FakeOptions: + hccl_config = None + + fake_torch_npu = ModuleType("torch_npu") + fake_torch_npu._C = SimpleNamespace( + _distributed_c10d=SimpleNamespace( + ProcessGroupHCCL=SimpleNamespace(Options=FakeOptions), + ), + ) + monkeypatch.setitem(sys.modules, "torch_npu", fake_torch_npu) + + options = create_hccl_process_group_options(2369) + other_options = create_hccl_process_group_options(2722) + + assert options.hccl_config == {"hccl_buffer_size": 2369} + assert other_options.hccl_config == {"hccl_buffer_size": 2722} + assert options is not other_options + + +def test_async_connector_logs_role_local_hccl_buffer(caplog): + with caplog.at_level(logging.INFO): + connector = CAMAsyncAFDConnector( + 0, + 0, + _vllm_config(extra_config={"attn_ranks_per_dp": 2}), + _afd_config(role="ffn"), + 0, + ) + + assert ( + connector.hccl_buffer_size_mb == connector.hccl_buffer_plan.ffn_buffer_size_mb + ) + assert "CAM async ffn HCCL buffer size is" in caplog.text + + def test_async_connector_factory_creates_import_safe_connector(): connector = AFDConnectorFactory.create_connector( 0, @@ -197,12 +237,10 @@ def test_async_connector_factory_creates_import_safe_connector(): assert isinstance(connector, CAMAsyncAFDConnector) assert not connector.is_initialized assert connector.control_plane is None - # tp_size is derived from the connector_extra_config attn_ranks_per_dp, - # which the factory reads through the same path as direct construction. - assert connector.tp_size == 2 + assert connector.num_npus_per_dp_group == 2 -def test_async_connector_uses_attn_ranks_per_dp_for_cam_tp_size(): +def test_async_connector_uses_attn_ranks_per_dp_for_npus_per_dp_group(): connector = CAMAsyncAFDConnector( 0, 0, @@ -215,7 +253,7 @@ def test_async_connector_uses_attn_ranks_per_dp_for_cam_tp_size(): 0, ) - assert connector.tp_size == 3 + assert connector.num_npus_per_dp_group == 3 @pytest.mark.parametrize("value", [True, "bad"]) @@ -257,6 +295,7 @@ def test_async_topology_uses_cam_attention_first_rank_layout(): def test_async_connector_init_creates_attention_first_hccl_group(monkeypatch): calls = [] + pg_options = object() fake_torch = _FakeTorch() monkeypatch.setattr(async_cam_module, "torch", fake_torch) monkeypatch.setattr( @@ -280,6 +319,11 @@ def fake_init_afd_process_group(**kwargs): "init_afd_process_group", fake_init_afd_process_group, ) + monkeypatch.setattr( + async_cam_module, + "create_hccl_process_group_options", + lambda buffer_size_mb: pg_options, + ) connector = CAMAsyncAFDConnector( 0, 0, @@ -298,6 +342,7 @@ def fake_init_afd_process_group(**kwargs): "rank": 5, "group_name": AFD_ASYNC_CAM_GROUP_NAME, "timeout": calls[0]["timeout"], + "pg_options": pg_options, }, ] assert connector.cam_pg is not None diff --git a/tests/unit/connectors/test_camp2p_connector.py b/tests/unit/connectors/test_camp2p_connector.py index 1abc3acb..33f045ea 100644 --- a/tests/unit/connectors/test_camp2p_connector.py +++ b/tests/unit/connectors/test_camp2p_connector.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import sys from types import ModuleType, SimpleNamespace @@ -49,7 +50,10 @@ def _vllm_config( tensor_parallel_size=1, num_ubatches=num_ubatches, ), - scheduler_config=SimpleNamespace(max_num_seqs=8), + scheduler_config=SimpleNamespace( + max_num_seqs=8, + max_num_batched_tokens=8, + ), model_config=SimpleNamespace( hf_config=SimpleNamespace( hidden_size=16, @@ -84,6 +88,22 @@ def test_camp2p_factory_creates_connector(): assert connector.extra_info.core_num == 12 +def test_camp2p_connector_logs_role_local_hccl_buffer(caplog): + with caplog.at_level(logging.INFO): + connector = CAMP2pAFDConnector( + 0, + 0, + _vllm_config(), + _afd_config(role="ffn"), + 0, + ) + + assert ( + connector.hccl_buffer_size_mb == connector.hccl_buffer_plan.ffn_buffer_size_mb + ) + assert "CAM P2P ffn HCCL buffer size is" in caplog.text + + def test_camp2p_topology_matches_original_rank_layout(): attn0 = build_camp2p_topology(_afd_config(role="attention"), 0) attn1 = build_camp2p_topology(_afd_config(role="attention"), 1) @@ -203,6 +223,7 @@ def test_camp2p_connector_uses_role_specific_core_num(monkeypatch): def test_camp2p_init_creates_one_hccl_group_per_ubatch(monkeypatch): calls = [] + options = [] monkeypatch.setitem(sys.modules, "torch_npu", ModuleType("torch_npu")) monkeypatch.setattr(camp2p_module, "ensure_cam_p2p_ops_available", lambda: None) @@ -223,6 +244,11 @@ def fake_init_afd_process_group(**kwargs): "init_afd_process_group", fake_init_afd_process_group, ) + monkeypatch.setattr( + camp2p_module, + "create_hccl_process_group_options", + lambda buffer_size_mb: options.append(buffer_size_mb) or object(), + ) connector = CAMP2pAFDConnector( 0, 0, @@ -234,6 +260,8 @@ def fake_init_afd_process_group(**kwargs): connector.init_afd_connector() assert [call["group_name"] for call in calls[:2]] == ["afd", "afd1"] + assert options == [connector.hccl_buffer_size_mb] * 2 + assert all(call["pg_options"] is not None for call in calls[:2]) assert connector.hccl_comm_name_list == ["hccl:afd:2", "hccl:afd1:2"] assert connector.hccl_comm_name == "hccl:afd:2" assert connector.hccl_comm_name2 == "hccl:afd1:2" @@ -257,6 +285,50 @@ def fake_init_afd_process_group(**kwargs): ) +def test_camp2p_ffn_applies_role_buffer_to_all_hccl_groups(monkeypatch): + calls = [] + options = [] + + monkeypatch.setitem(sys.modules, "torch_npu", ModuleType("torch_npu")) + monkeypatch.setattr(camp2p_module, "ensure_cam_p2p_ops_available", lambda: None) + monkeypatch.setattr(camp2p_module, "_register_camp2p_custom_ops", lambda: None) + + def fake_init_afd_process_group(**kwargs): + calls.append(kwargs) + backend = SimpleNamespace( + get_hccl_comm_name=lambda rank: f"hccl:{kwargs['group_name']}:{rank}", + ) + return SimpleNamespace( + group_name=kwargs["group_name"], + _get_backend=lambda device: backend, + ) + + monkeypatch.setattr( + camp2p_module, + "init_afd_process_group", + fake_init_afd_process_group, + ) + monkeypatch.setattr( + camp2p_module, + "create_hccl_process_group_options", + lambda buffer_size_mb: options.append(buffer_size_mb) or object(), + ) + connector = CAMP2pAFDConnector( + 0, + 0, + _vllm_config(), + _afd_config(role="ffn"), + 0, + ) + + connector.init_afd_connector() + + assert [call["group_name"] for call in calls] == ["afd", "afd_moe", "p2p"] + assert options == [connector.hccl_buffer_size_mb] * 2 + assert all(call["pg_options"] is not None for call in calls[:2]) + assert "pg_options" not in calls[2] + + def test_camp2p_send_attn_custom_op_receives_all_hccl_names(monkeypatch): torch = pytest.importorskip("torch") captured = {} diff --git a/tests/unit/distributed/test_cam_hccl_buffer.py b/tests/unit/distributed/test_cam_hccl_buffer.py new file mode 100644 index 00000000..8d933a1c --- /dev/null +++ b/tests/unit/distributed/test_cam_hccl_buffer.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import logging +from types import SimpleNamespace + +import pytest + +from afd_plugin.config import AFDConfig +from afd_plugin.distributed.cam_hccl_buffer import ( + derive_cam_hccl_buffer_plan, + derive_cam_hccl_buffer_plan_from_config, + warn_if_cam_memory_headroom_is_low, +) + + +def _vllm_config( + *, + connector: str = "CAMAsyncAFDConnector", + num_npus_per_dp_group: int = 8, + dynamic_quant: int = 1, + gpu_memory_utilization: float = 0.9, +): + extra_config = ( + { + "attn_ranks_per_dp": num_npus_per_dp_group, + "dynamicQuant": dynamic_quant, + } + if connector == "CAMAsyncAFDConnector" + else {} + ) + return SimpleNamespace( + additional_config={"afd": {"connector_extra_config": extra_config}}, + parallel_config=SimpleNamespace( + tensor_parallel_size=num_npus_per_dp_group, + ), + scheduler_config=SimpleNamespace(max_num_batched_tokens=140000), + cache_config=SimpleNamespace( + gpu_memory_utilization=gpu_memory_utilization, + ), + model_config=SimpleNamespace( + hf_config=SimpleNamespace( + hidden_size=7168, + num_experts_per_tok=8, + ), + ), + ) + + +def _afd_config(*, connector: str, role: str) -> AFDConfig: + return AFDConfig( + connector=connector, + role=role, + num_attention_ranks=24, + num_ffn_ranks=8, + ) + + +def test_deepseek_v32_cam_buffers_are_derived_independently_by_role(): + plan = derive_cam_hccl_buffer_plan( + hidden_size=7168, + max_batch_tokens=140000, + num_npus_per_dp_group=8, + topk=8, + attention_rank_size=24, + dynamic_quant=1, + ) + + assert plan.attention_required_bytes == 2_257_920_000 + assert plan.moe_required_bytes == 2_593_920_000 + assert plan.attention_buffer_size_mb == 2369 + assert plan.ffn_buffer_size_mb == 2722 + assert plan.buffer_size_mb_for_role("attention") == 2369 + assert plan.buffer_size_mb_for_role("ffn") == 2722 + + +def test_moe_buffer_has_no_topk_multiplier_and_uses_non_quantized_width(): + low_topk = derive_cam_hccl_buffer_plan( + hidden_size=7168, + max_batch_tokens=140000, + num_npus_per_dp_group=8, + topk=1, + attention_rank_size=24, + dynamic_quant=0, + ) + high_topk = derive_cam_hccl_buffer_plan( + hidden_size=7168, + max_batch_tokens=140000, + num_npus_per_dp_group=8, + topk=8, + attention_rank_size=24, + dynamic_quant=0, + ) + + assert low_topk.moe_required_bytes == 5_160_960_000 + assert high_topk.moe_required_bytes == low_topk.moe_required_bytes + assert high_topk.ffn_buffer_size_mb == 5415 + assert high_topk.attention_required_bytes > low_topk.attention_required_bytes + + +def test_cam_buffer_rounds_partial_per_npu_token_and_mb_up(): + plan = derive_cam_hccl_buffer_plan( + hidden_size=3, + max_batch_tokens=5, + num_npus_per_dp_group=2, + topk=1, + attention_rank_size=1, + dynamic_quant=1, + ) + + assert plan.attention_required_bytes == 36 + assert plan.moe_required_bytes == 18_528 + assert plan.attention_buffer_size_mb == 1 + assert plan.ffn_buffer_size_mb == 1 + + +def test_buffer_plan_from_config_supports_async_and_camp2p(): + async_plan = derive_cam_hccl_buffer_plan_from_config( + _vllm_config(), + _afd_config(connector="CAMAsyncAFDConnector", role="attention"), + ) + camp2p_plan = derive_cam_hccl_buffer_plan_from_config( + _vllm_config( + connector="CAMP2pAFDConnector", + dynamic_quant=0, + ), + _afd_config(connector="CAMP2pAFDConnector", role="ffn"), + ) + + assert async_plan.attention_buffer_size_mb == 2369 + assert async_plan.ffn_buffer_size_mb == 2722 + assert camp2p_plan.attention_buffer_size_mb == 2369 + assert camp2p_plan.ffn_buffer_size_mb == 5415 + + +@pytest.mark.parametrize( + "connector", + ["CAMAsyncAFDConnector", "CAMP2pAFDConnector"], +) +def test_cam_memory_headroom_warns_without_adjusting_utilization( + connector, + caplog, +): + vllm_config = _vllm_config( + connector=connector, + gpu_memory_utilization=0.95, + ) + + with caplog.at_level(logging.WARNING): + warn_if_cam_memory_headroom_is_low( + vllm_config, + _afd_config(connector=connector, role="attention"), + 64 * 1024**3, + ) + + assert vllm_config.cache_config.gpu_memory_utilization == 0.95 + assert "consider setting gpu_memory_utilization" in caplog.text + assert "configured value 0.950000 is unchanged" in caplog.text + + +def test_cam_memory_headroom_does_not_warn_when_already_safe(caplog): + vllm_config = _vllm_config(gpu_memory_utilization=0.75) + + with caplog.at_level(logging.WARNING): + warn_if_cam_memory_headroom_is_low( + vllm_config, + _afd_config(connector="CAMAsyncAFDConnector", role="attention"), + 64 * 1024**3, + ) + + assert caplog.text == "" + + +def test_cam_buffer_rejects_unsupported_role_and_dynamic_quant(): + plan = derive_cam_hccl_buffer_plan( + hidden_size=16, + max_batch_tokens=8, + num_npus_per_dp_group=1, + topk=2, + attention_rank_size=4, + dynamic_quant=1, + ) + + with pytest.raises(ValueError, match="unsupported AFD role"): + plan.buffer_size_mb_for_role("decode") + with pytest.raises(ValueError, match="dynamic_quant must be 0 or 1"): + derive_cam_hccl_buffer_plan( + hidden_size=16, + max_batch_tokens=8, + num_npus_per_dp_group=1, + topk=2, + attention_rank_size=4, + dynamic_quant=2, + ) + with pytest.raises(ValueError, match="num_npus_per_dp_group must be positive"): + derive_cam_hccl_buffer_plan( + hidden_size=16, + max_batch_tokens=8, + num_npus_per_dp_group=0, + topk=2, + attention_rank_size=4, + dynamic_quant=1, + ) diff --git a/tests/unit/test_e2e_runner.py b/tests/unit/test_e2e_runner.py index 3010fc5d..0379aa12 100644 --- a/tests/unit/test_e2e_runner.py +++ b/tests/unit/test_e2e_runner.py @@ -1201,3 +1201,13 @@ def test_runner_forces_gpu_v1_model_runner(monkeypatch): env = runner.build_env("0,1", args, role="attention") assert env["VLLM_USE_V2_MODEL_RUNNER"] == "0" + + +def test_runner_preserves_global_hccl_buffer(monkeypatch): + args = _args() + args.device_backend = "npu" + monkeypatch.setenv("HCCL_BUFFSIZE", "8192") + + env = runner.build_env("0,1", args, role="attention") + + assert env["HCCL_BUFFSIZE"] == "8192" diff --git a/tests/unit/v1/worker/test_npu_device_contract.py b/tests/unit/v1/worker/test_npu_device_contract.py index ff673315..6eee70b8 100644 --- a/tests/unit/v1/worker/test_npu_device_contract.py +++ b/tests/unit/v1/worker/test_npu_device_contract.py @@ -14,3 +14,25 @@ def test_npu_connectors_use_model_device_index_for_dp_workers(): assert "rank, _ = _resolve_world_ranks()" in source assert "local_rank = int(device.index)" in source assert "rank, local_rank = _resolve_world_ranks()" not in source + + +def test_cam_memory_headroom_check_does_not_adjust_utilization(): + for source_path in ( + Path("afd_plugin/v1/worker/npu/attention_worker.py"), + Path("afd_plugin/v1/worker/npu/ffn_worker.py"), + ): + source = source_path.read_text() + + assert "warn_if_cam_memory_headroom_is_low(" in source + assert "self.init_snapshot.total_memory" in source + assert "self.requested_memory = (" not in source + + +def test_cam_workers_do_not_modify_global_hccl_buffer_configuration(): + for source_path in ( + Path("afd_plugin/v1/worker/npu/attention_worker.py"), + Path("afd_plugin/v1/worker/npu/ffn_worker.py"), + ): + source = source_path.read_text() + + assert "HCCL_BUFFSIZE" not in source