Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions lib/server.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ start_server() {
"$image" \
"$model" --port "$port" $serve_args

# Install pytest to avoid cupy.testing import failure during torch.compile
docker exec "$container" pip install -q pytest 2>/dev/null || true

echo "--- :memo: streaming vllm logs"
( docker logs -f "$container" 2>&1 | stdbuf -oL -eL sed 's/^/[vllm] /' ) &
VLLM_LOGS_PID=$!
Expand Down
26 changes: 16 additions & 10 deletions workloads/deepseek_v4_flash_b200.yaml
Original file line number Diff line number Diff line change
@@ -1,27 +1,33 @@
# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe, MTP spec-decode)
# DeepSeek-V4-Flash on B200 (TP=8 + EP, deep_gemm_mega_moe)
name: deepseek_v4_flash-b200
gpu: B200
num_gpus: 8
nightly: true

vllm:
model: deepseek-ai/DeepSeek-V4-Flash
env:
# The vLLM release image's nvidia-cu13 wheels are minor-skewed (nvcc minor
# != cudart-headers minor, both CUDA 13), so the DeepSeek-V4 mHC TileLang
# kernel's sm_100a JIT compile trips CCCL's nvcc-vs-headers guard ("CUDA
# compiler and CUDA toolkit headers are incompatible"). nvcc honors
# NVCC_APPEND_FLAGS and TileLang's compile subprocess inherits the env, so
# define CCCL's official opt-out to skip the (same-major) check.
# Drop this once the image ships version-aligned cu13 wheels.
# See vllm-project/vllm#44305.
NVCC_APPEND_FLAGS: "-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK"
serve_args: >-
--tensor-parallel-size 2
--data-parallel-size 4
--enable-expert-parallel
--max-model-len 32768
--trust-remote-code
--kv-cache-dtype fp8
--block-size 256
--moe-backend deep_gemm_mega_moe
--enable-expert-parallel
--tensor-parallel-size 8
--attention_config.use_fp4_indexer_cache=True
--moe-backend deep_gemm_mega_moe
--tokenizer-mode deepseek_v4
--tool-call-parser deepseek_v4
--reasoning-parser deepseek_v4
--enable-auto-tool-choice
--trust-remote-code
--speculative_config.method=mtp
--speculative_config.num_speculative_tokens=2
--reasoning-parser deepseek_v4

bfcl:
test_categories:
Expand Down