From 6cdb1cfec12ecb87a0d5153acd4910a68728eaf9 Mon Sep 17 00:00:00 2001 From: khluu Date: Thu, 28 May 2026 14:34:13 -0700 Subject: [PATCH 1/8] Add B200 workload variants for all H200-only models Co-Authored-By: Claude Opus 4.6 (1M context) --- workloads/deepseek_v3_2_b200.yaml | 47 ++++++++++++++++++++++++ workloads/deepseek_v4_pro_5_b200.yaml | 49 ++++++++++++++++++++++++++ workloads/glm_5_1_b200.yaml | 43 ++++++++++++++++++++++ workloads/kimi_k2_5_b200.yaml | 42 ++++++++++++++++++++++ workloads/minimax_m2_5_b200.yaml | 43 ++++++++++++++++++++++ workloads/nemotron_3_super_5_b200.yaml | 45 +++++++++++++++++++++++ workloads/qwen3_5_b200.yaml | 43 ++++++++++++++++++++++ 7 files changed, 312 insertions(+) create mode 100644 workloads/deepseek_v3_2_b200.yaml create mode 100644 workloads/deepseek_v4_pro_5_b200.yaml create mode 100644 workloads/glm_5_1_b200.yaml create mode 100644 workloads/kimi_k2_5_b200.yaml create mode 100644 workloads/minimax_m2_5_b200.yaml create mode 100644 workloads/nemotron_3_super_5_b200.yaml create mode 100644 workloads/qwen3_5_b200.yaml diff --git a/workloads/deepseek_v3_2_b200.yaml b/workloads/deepseek_v3_2_b200.yaml new file mode 100644 index 0000000..c53b520 --- /dev/null +++ b/workloads/deepseek_v3_2_b200.yaml @@ -0,0 +1,47 @@ +# DeepSeek-V3.2 on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe) +name: deepseek_v3_2-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: deepseek-ai/DeepSeek-V3.2 + serve_args: >- + --tensor-parallel-size 2 + --data-parallel-size 4 + --enable-expert-parallel + --max-model-len 32768 + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm_mega_moe + --attention_config.use_fp4_indexer_cache=True + --tokenizer-mode deepseek_v32 + --tool-call-parser deepseek_v32 + --reasoning-parser deepseek_v3 + --enable-auto-tool-choice + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 4096 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/deepseek_v4_pro_5_b200.yaml b/workloads/deepseek_v4_pro_5_b200.yaml new file mode 100644 index 0000000..c3cf903 --- /dev/null +++ b/workloads/deepseek_v4_pro_5_b200.yaml @@ -0,0 +1,49 @@ +# DeepSeek-V4-Pro on B200 (DP=8 + EP, deep_gemm_mega_moe) +name: deepseek_v4_pro-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: deepseek-ai/DeepSeek-V4-Pro + serve_args: >- + --data-parallel-size 8 + --enable-expert-parallel + --max-model-len 32768 + --max-num-seqs 512 + --max-num-batched-tokens 512 + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm_mega_moe + --attention_config.use_fp4_indexer_cache=True + --tokenizer-mode deepseek_v4 + --gpu-memory-utilization 0.95 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/glm_5_1_b200.yaml b/workloads/glm_5_1_b200.yaml new file mode 100644 index 0000000..9d13380 --- /dev/null +++ b/workloads/glm_5_1_b200.yaml @@ -0,0 +1,43 @@ +# GLM-5.1 (FP8) on B200 +name: glm_5_1-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: zai-org/GLM-5.1-FP8 + serve_args: >- + --tensor-parallel-size 8 + --kv-cache-dtype fp8 + --block-size 256 + --speculative-config.method mtp + --speculative-config.num_speculative_tokens 3 + --tool-call-parser glm47 + --reasoning-parser glm45 + --enable-auto-tool-choice + --chat-template-content-format=string + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/kimi_k2_5_b200.yaml b/workloads/kimi_k2_5_b200.yaml new file mode 100644 index 0000000..09d143c --- /dev/null +++ b/workloads/kimi_k2_5_b200.yaml @@ -0,0 +1,42 @@ +# Kimi-K2.5 on B200 +name: kimi_k2_5-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: moonshotai/Kimi-K2.5 + serve_args: >- + --tensor-parallel-size 8 + --kv-cache-dtype fp8 + --block-size 256 + --mm-encoder-tp-mode data + --tool-call-parser kimi_k2 + --reasoning-parser kimi_k2 + --enable-auto-tool-choice + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/minimax_m2_5_b200.yaml b/workloads/minimax_m2_5_b200.yaml new file mode 100644 index 0000000..4a9c351 --- /dev/null +++ b/workloads/minimax_m2_5_b200.yaml @@ -0,0 +1,43 @@ +# MiniMax-M2.5 on B200 (TP=4 + EP, deep_gemm_mega_moe) +name: minimax_m2_5-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: MiniMaxAI/MiniMax-M2.5 + serve_args: >- + --tensor-parallel-size 4 + --enable-expert-parallel + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm_mega_moe + --tool-call-parser minimax_m2 + --reasoning-parser minimax_m2 + --enable-auto-tool-choice + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/nemotron_3_super_5_b200.yaml b/workloads/nemotron_3_super_5_b200.yaml new file mode 100644 index 0000000..3af1527 --- /dev/null +++ b/workloads/nemotron_3_super_5_b200.yaml @@ -0,0 +1,45 @@ +# Nemotron-3-Super-120B-A12B (FP8) on B200 +# Hybrid Mamba-2 + LatentMoE architecture (120B total, 12B active). +# TP=8 + EP; MTP speculative decoding enabled. +name: nemotron_3_super-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 + serve_args: >- + --tensor-parallel-size 8 + --enable-expert-parallel + --max-model-len 32768 + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm_mega_moe + --speculative-config.method mtp + --speculative-config.num_speculative_tokens 5 + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/qwen3_5_b200.yaml b/workloads/qwen3_5_b200.yaml new file mode 100644 index 0000000..eec4155 --- /dev/null +++ b/workloads/qwen3_5_b200.yaml @@ -0,0 +1,43 @@ +# Qwen3.5 on B200 (TP=8 + EP, deep_gemm_mega_moe) +name: qwen3_5-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: Qwen/Qwen3.5-397B-A17B-FP8 + serve_args: >- + --tensor-parallel-size 8 + --enable-expert-parallel + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm_mega_moe + --reasoning-parser qwen3 + --language-model-only + --trust-remote-code + --max-num-seqs 256 + --max-model-len auto + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From 47f0c1621807cc3e33fc927806d4635b3e56f235 Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 29 May 2026 02:11:31 -0700 Subject: [PATCH 2/8] Use deep_gemm MoE backend for B200 workloads deep_gemm_mega_moe is not yet available in the current nightly image. Co-Authored-By: Claude Opus 4.6 (1M context) --- workloads/deepseek_v3_2_b200.yaml | 4 ++-- workloads/deepseek_v4_flash_b200.yaml | 4 ++-- workloads/deepseek_v4_pro_5_b200.yaml | 4 ++-- workloads/minimax_m2_5_b200.yaml | 4 ++-- workloads/nemotron_3_super_5_b200.yaml | 2 +- workloads/qwen3_5_b200.yaml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/workloads/deepseek_v3_2_b200.yaml b/workloads/deepseek_v3_2_b200.yaml index c53b520..635c3bb 100644 --- a/workloads/deepseek_v3_2_b200.yaml +++ b/workloads/deepseek_v3_2_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V3.2 on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe) +# DeepSeek-V3.2 on B200 (TP=2 × DP=4 + EP, deep_gemm) name: deepseek_v3_2-b200 gpu: B200 num_gpus: 8 @@ -13,7 +13,7 @@ vllm: --max-model-len 32768 --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v32 --tool-call-parser deepseek_v32 diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index 694ab37..34ac08b 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe, MTP spec-decode) +# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm, MTP spec-decode) name: deepseek_v4_flash-b200 gpu: B200 num_gpus: 8 @@ -13,7 +13,7 @@ vllm: --max-model-len 32768 --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 diff --git a/workloads/deepseek_v4_pro_5_b200.yaml b/workloads/deepseek_v4_pro_5_b200.yaml index c3cf903..92b4cf8 100644 --- a/workloads/deepseek_v4_pro_5_b200.yaml +++ b/workloads/deepseek_v4_pro_5_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Pro on B200 (DP=8 + EP, deep_gemm_mega_moe) +# DeepSeek-V4-Pro on B200 (DP=8 + EP, deep_gemm) name: deepseek_v4_pro-b200 gpu: B200 num_gpus: 8 @@ -14,7 +14,7 @@ vllm: --max-num-batched-tokens 512 --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --gpu-memory-utilization 0.95 diff --git a/workloads/minimax_m2_5_b200.yaml b/workloads/minimax_m2_5_b200.yaml index 4a9c351..2ab5694 100644 --- a/workloads/minimax_m2_5_b200.yaml +++ b/workloads/minimax_m2_5_b200.yaml @@ -1,4 +1,4 @@ -# MiniMax-M2.5 on B200 (TP=4 + EP, deep_gemm_mega_moe) +# MiniMax-M2.5 on B200 (TP=4 + EP, deep_gemm) name: minimax_m2_5-b200 gpu: B200 num_gpus: 8 @@ -11,7 +11,7 @@ vllm: --enable-expert-parallel --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --tool-call-parser minimax_m2 --reasoning-parser minimax_m2 --enable-auto-tool-choice diff --git a/workloads/nemotron_3_super_5_b200.yaml b/workloads/nemotron_3_super_5_b200.yaml index 3af1527..4be2ff7 100644 --- a/workloads/nemotron_3_super_5_b200.yaml +++ b/workloads/nemotron_3_super_5_b200.yaml @@ -14,7 +14,7 @@ vllm: --max-model-len 32768 --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --speculative-config.method mtp --speculative-config.num_speculative_tokens 5 --trust-remote-code diff --git a/workloads/qwen3_5_b200.yaml b/workloads/qwen3_5_b200.yaml index eec4155..5d0f147 100644 --- a/workloads/qwen3_5_b200.yaml +++ b/workloads/qwen3_5_b200.yaml @@ -1,4 +1,4 @@ -# Qwen3.5 on B200 (TP=8 + EP, deep_gemm_mega_moe) +# Qwen3.5 on B200 (TP=8 + EP, deep_gemm) name: qwen3_5-b200 gpu: B200 num_gpus: 8 @@ -11,7 +11,7 @@ vllm: --enable-expert-parallel --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --reasoning-parser qwen3 --language-model-only --trust-remote-code From 9c52d6409789a630a63ee386b1f13f70d4cf12bb Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 29 May 2026 02:36:01 -0700 Subject: [PATCH 3/8] Switch B200 workloads to NVFP4 models and match vLLM recipes Use NVIDIA NVFP4 model variants and official vLLM recipe serve args for B200 workloads (recipes.vllm.ai). Key changes: - deepseek_v3_2: NVFP4 + FLASHINFER_MLA attention - qwen3_5: NVFP4 + flashinfer MoE FP4 - nemotron_3_super: NVFP4, TP=1 - glm_5_1: NVFP4 + flashinfer MoE FP4 - kimi_k2_5: NVFP4 + Eagle3 speculative decoding - minimax_m2_5: match B200 recipe (TP=4, compilation config) - deepseek_v4_flash: unchanged (already correct) Co-Authored-By: Claude Opus 4.6 (1M context) --- workloads/deepseek_v3_2_b200.yaml | 21 ++++++++++----------- workloads/glm_5_1_b200.yaml | 17 +++++++++-------- workloads/kimi_k2_5_b200.yaml | 18 +++++++++++------- workloads/minimax_m2_5_b200.yaml | 12 +++++------- workloads/nemotron_3_super_5_b200.yaml | 19 +++++++------------ workloads/qwen3_5_b200.yaml | 23 +++++++++++++---------- 6 files changed, 55 insertions(+), 55 deletions(-) diff --git a/workloads/deepseek_v3_2_b200.yaml b/workloads/deepseek_v3_2_b200.yaml index 635c3bb..eae34bb 100644 --- a/workloads/deepseek_v3_2_b200.yaml +++ b/workloads/deepseek_v3_2_b200.yaml @@ -1,25 +1,24 @@ -# DeepSeek-V3.2 on B200 (TP=2 × DP=4 + EP, deep_gemm) +# DeepSeek-V3.2 NVFP4 on B200 (TP=8, FLASHINFER_MLA) +# Recipe: https://recipes.vllm.ai/nvidia/DeepSeek-V3.2-NVFP4 name: deepseek_v3_2-b200 gpu: B200 num_gpus: 8 nightly: true vllm: - model: deepseek-ai/DeepSeek-V3.2 + model: nvidia/DeepSeek-V3.2-NVFP4 serve_args: >- - --tensor-parallel-size 2 - --data-parallel-size 4 - --enable-expert-parallel - --max-model-len 32768 + --trust-remote-code + --kernel-config.enable_flashinfer_autotune=False --kv-cache-dtype fp8 - --block-size 256 - --moe-backend deep_gemm - --attention_config.use_fp4_indexer_cache=True + --tensor-parallel-size 8 + --attention-backend FLASHINFER_MLA --tokenizer-mode deepseek_v32 --tool-call-parser deepseek_v32 - --reasoning-parser deepseek_v3 --enable-auto-tool-choice - --trust-remote-code + --reasoning-parser deepseek_v3 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" lm_eval: model_args: diff --git a/workloads/glm_5_1_b200.yaml b/workloads/glm_5_1_b200.yaml index 9d13380..4ddc6e3 100644 --- a/workloads/glm_5_1_b200.yaml +++ b/workloads/glm_5_1_b200.yaml @@ -1,21 +1,22 @@ -# GLM-5.1 (FP8) on B200 +# GLM-5.1 NVFP4 on B200 (TP=8) +# Recipe: https://recipes.vllm.ai/nvidia/GLM-5.1-NVFP4 name: glm_5_1-b200 gpu: B200 num_gpus: 8 nightly: true vllm: - model: zai-org/GLM-5.1-FP8 + model: nvidia/GLM-5.1-NVFP4 serve_args: >- - --tensor-parallel-size 8 + --trust-remote-code + --chat-template-content-format=string --kv-cache-dtype fp8 - --block-size 256 - --speculative-config.method mtp - --speculative-config.num_speculative_tokens 3 + --tensor-parallel-size 8 --tool-call-parser glm47 - --reasoning-parser glm45 --enable-auto-tool-choice - --chat-template-content-format=string + --reasoning-parser glm45 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" lm_eval: model_args: diff --git a/workloads/kimi_k2_5_b200.yaml b/workloads/kimi_k2_5_b200.yaml index 09d143c..e877b1d 100644 --- a/workloads/kimi_k2_5_b200.yaml +++ b/workloads/kimi_k2_5_b200.yaml @@ -1,20 +1,24 @@ -# Kimi-K2.5 on B200 +# Kimi-K2.5 NVFP4 on B200 (TP=8, Eagle3 speculative) +# Recipe: https://recipes.vllm.ai/nvidia/Kimi-K2.5-NVFP4 name: kimi_k2_5-b200 gpu: B200 num_gpus: 8 nightly: true vllm: - model: moonshotai/Kimi-K2.5 + model: nvidia/Kimi-K2.5-NVFP4 serve_args: >- - --tensor-parallel-size 8 + --trust-remote-code --kv-cache-dtype fp8 - --block-size 256 - --mm-encoder-tp-mode data + --tensor-parallel-size 8 + --attention-config.use_trtllm_ragged_deepseek_prefill=True --tool-call-parser kimi_k2 - --reasoning-parser kimi_k2 --enable-auto-tool-choice - --trust-remote-code + --reasoning-parser kimi_k2 + --speculative-config '{"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3}' + --mm-encoder-tp-mode data + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" lm_eval: model_args: diff --git a/workloads/minimax_m2_5_b200.yaml b/workloads/minimax_m2_5_b200.yaml index 2ab5694..21bc4dc 100644 --- a/workloads/minimax_m2_5_b200.yaml +++ b/workloads/minimax_m2_5_b200.yaml @@ -1,4 +1,5 @@ -# MiniMax-M2.5 on B200 (TP=4 + EP, deep_gemm) +# MiniMax-M2.5 on B200 (TP=4) +# Recipe: https://recipes.vllm.ai/MiniMaxAI/MiniMax-M2.5 name: minimax_m2_5-b200 gpu: B200 num_gpus: 8 @@ -7,15 +8,12 @@ nightly: true vllm: model: MiniMaxAI/MiniMax-M2.5 serve_args: >- + --trust-remote-code --tensor-parallel-size 4 - --enable-expert-parallel - --kv-cache-dtype fp8 - --block-size 256 - --moe-backend deep_gemm --tool-call-parser minimax_m2 - --reasoning-parser minimax_m2 --enable-auto-tool-choice - --trust-remote-code + --reasoning-parser minimax_m2 + --compilation-config '{"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}}' lm_eval: model_args: diff --git a/workloads/nemotron_3_super_5_b200.yaml b/workloads/nemotron_3_super_5_b200.yaml index 4be2ff7..2b72450 100644 --- a/workloads/nemotron_3_super_5_b200.yaml +++ b/workloads/nemotron_3_super_5_b200.yaml @@ -1,23 +1,18 @@ -# Nemotron-3-Super-120B-A12B (FP8) on B200 -# Hybrid Mamba-2 + LatentMoE architecture (120B total, 12B active). -# TP=8 + EP; MTP speculative decoding enabled. +# Nemotron-3-Super-120B-A12B NVFP4 on B200 (TP=1) +# Recipe: https://recipes.vllm.ai/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 name: nemotron_3_super-b200 gpu: B200 num_gpus: 8 nightly: true vllm: - model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 + model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 serve_args: >- - --tensor-parallel-size 8 - --enable-expert-parallel - --max-model-len 32768 - --kv-cache-dtype fp8 - --block-size 256 - --moe-backend deep_gemm - --speculative-config.method mtp - --speculative-config.num_speculative_tokens 5 --trust-remote-code + --tensor-parallel-size 1 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --reasoning-parser nemotron_v3 lm_eval: model_args: diff --git a/workloads/qwen3_5_b200.yaml b/workloads/qwen3_5_b200.yaml index 5d0f147..3db44bc 100644 --- a/workloads/qwen3_5_b200.yaml +++ b/workloads/qwen3_5_b200.yaml @@ -1,22 +1,25 @@ -# Qwen3.5 on B200 (TP=8 + EP, deep_gemm) +# Qwen3.5-397B-A17B NVFP4 on B200 (TP=8) +# Recipe: https://recipes.vllm.ai/nvidia/Qwen3.5-397B-A17B-NVFP4 name: qwen3_5-b200 gpu: B200 num_gpus: 8 nightly: true vllm: - model: Qwen/Qwen3.5-397B-A17B-FP8 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 serve_args: >- - --tensor-parallel-size 8 - --enable-expert-parallel + --trust-remote-code --kv-cache-dtype fp8 - --block-size 256 - --moe-backend deep_gemm + --tensor-parallel-size 8 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder --reasoning-parser qwen3 - --language-model-only - --trust-remote-code - --max-num-seqs 256 - --max-model-len auto + --mm-encoder-tp-mode data + env: + VLLM_DEEP_GEMM_WARMUP: skip + VLLM_USE_DEEP_GEMM: "0" + VLLM_FLASHINFER_MOE_BACKEND: latency + VLLM_USE_FLASHINFER_MOE_FP4: "1" lm_eval: model_args: From 61e594d9f83b38b928207047bec9a0431ca51b2c Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 29 May 2026 03:45:16 -0700 Subject: [PATCH 4/8] Fix JSON serve_args quoting for kimi/minimax B200 workloads server.sh word-splits $serve_args unquoted, so bash does not strip the single quotes wrapping the --speculative-config / --compilation-config JSON. vLLM received the literal '{...}' (quotes included) and rejected it with "cannot be converted". The compact JSON has no spaces, so dropping the surrounding single quotes lets word-splitting yield one valid token. AI-assisted change. Co-Authored-By: Claude --- workloads/kimi_k2_5_b200.yaml | 2 +- workloads/minimax_m2_5_b200.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workloads/kimi_k2_5_b200.yaml b/workloads/kimi_k2_5_b200.yaml index e877b1d..d04899f 100644 --- a/workloads/kimi_k2_5_b200.yaml +++ b/workloads/kimi_k2_5_b200.yaml @@ -15,7 +15,7 @@ vllm: --tool-call-parser kimi_k2 --enable-auto-tool-choice --reasoning-parser kimi_k2 - --speculative-config '{"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3}' + --speculative-config {"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3} --mm-encoder-tp-mode data env: VLLM_USE_FLASHINFER_MOE_FP4: "1" diff --git a/workloads/minimax_m2_5_b200.yaml b/workloads/minimax_m2_5_b200.yaml index 21bc4dc..053c65d 100644 --- a/workloads/minimax_m2_5_b200.yaml +++ b/workloads/minimax_m2_5_b200.yaml @@ -13,7 +13,7 @@ vllm: --tool-call-parser minimax_m2 --enable-auto-tool-choice --reasoning-parser minimax_m2 - --compilation-config '{"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}}' + --compilation-config {"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}} lm_eval: model_args: From c96769bf1887c7364476c8fdc9f1f171cad06a87 Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 29 May 2026 03:45:41 -0700 Subject: [PATCH 5/8] docs: note serve_args word-split is unquoted (no quoted/space-bearing JSON) AI-assisted change. Co-Authored-By: Claude --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 93bfb1b..a085f0f 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,9 @@ vllm: # how the server is brought up env: # optional; merged over the GPU profile's env SOME_VAR: value serve_args: >- # appended to `vllm serve `; word-split - -dp 8 --enable-expert-parallel - --trust-remote-code + -dp 8 --enable-expert-parallel # NOTE: word-split is unquoted, so do NOT wrap + --trust-remote-code # values in quotes and keep each token space-free. + --speculative-config {"method":"eagle3","num_speculative_tokens":3} # JSON args: no quotes, no spaces lm_eval: # accuracy tasks (optional) model_args: # workload-level defaults, merged into every task From 3aa066c506951e1341be41e65b6fde168fc26ec9 Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 29 May 2026 03:53:55 -0700 Subject: [PATCH 6/8] Fix deepseek_v3_2 B200: drop forced FLASHINFER_MLA backend DeepSeek-V3.2 uses sparse attention; forcing --attention-backend FLASHINFER_MLA crashes engine init with "Selected backend AttentionBackendEnum.FLASHINFER_MLA is not valid ... ['sparse not supported']". The vLLM recipe sets no --attention-backend, letting vLLM auto-select a sparse-capable MLA backend. Remove the override. AI-assisted change. Co-Authored-By: Claude --- workloads/deepseek_v3_2_b200.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workloads/deepseek_v3_2_b200.yaml b/workloads/deepseek_v3_2_b200.yaml index eae34bb..574699a 100644 --- a/workloads/deepseek_v3_2_b200.yaml +++ b/workloads/deepseek_v3_2_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V3.2 NVFP4 on B200 (TP=8, FLASHINFER_MLA) +# DeepSeek-V3.2 NVFP4 on B200 (TP=8; sparse attention, auto MLA backend) # Recipe: https://recipes.vllm.ai/nvidia/DeepSeek-V3.2-NVFP4 name: deepseek_v3_2-b200 gpu: B200 @@ -12,7 +12,6 @@ vllm: --kernel-config.enable_flashinfer_autotune=False --kv-cache-dtype fp8 --tensor-parallel-size 8 - --attention-backend FLASHINFER_MLA --tokenizer-mode deepseek_v32 --tool-call-parser deepseek_v32 --enable-auto-tool-choice From df8929aa1bbc8abb8a4ab0509c082a24f5595c55 Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 29 May 2026 14:55:00 -0700 Subject: [PATCH 7/8] Cache B200 models on /raid instead of root disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /mnt/shared is NOT a shared mount on the DGX B200 nodes — it's a plain directory on each node's ~1.8T root disk. With HF_HOME=/mnt/shared/hf_cache, model weights accumulated on root until the kubelet hit disk-pressure and evicted job pods ~20s into setup (seen as "signal: terminated" on dgxb200-11). Point the B200 HF cache at the 28T /raid volume (already mounted into the pod) in both lib/gpu_profiles.yaml (authoritative, exported as HF_HOME by parse_workload) and the K8s pod env in generate_pipeline.py. AI-assisted change. Co-Authored-By: Claude --- .buildkite/generate_pipeline.py | 2 +- lib/gpu_profiles.yaml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 82d3703..ce6d8bf 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -114,7 +114,7 @@ def b200_k8s_plugin(image, num_gpus): "env": [ {"name": "VLLM_USAGE_SOURCE", "value": "ci-test"}, {"name": "NCCL_CUMEM_HOST_ENABLE", "value": "0"}, - {"name": "HF_HOME", "value": "/mnt/shared/hf_cache"}, + {"name": "HF_HOME", "value": "/raid/hf_cache"}, { "name": "HF_TOKEN", "valueFrom": { diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 329080c..ce285c2 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -8,7 +8,10 @@ H200: B200: queue: b200-k8s - hf_home: /mnt/shared/hf_cache + # Cache models on the 28T /raid volume, not the ~1.8T root disk. /mnt/shared + # is NOT a shared mount on the DGX B200 nodes — it's a plain dir on root, so + # caching there fills root and triggers kubelet disk-pressure pod eviction. + hf_home: /raid/hf_cache server_runtime: native env: VLLM_DEEP_GEMM_WARMUP: skip From a019b5598dee0857357182235e84649b60a2e3d9 Mon Sep 17 00:00:00 2001 From: khluu Date: Mon, 1 Jun 2026 01:08:31 -0700 Subject: [PATCH 8/8] Fix kimi attention-config arg and native vllm-bench pandas dep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kimi: --attention-config.use_trtllm_ragged_deepseek_prefill is not a valid AttentionConfig field. The TRT-LLM ragged DeepSeek prefill path is selected via the MLA prefill backend, so use --attention-config.mla_prefill_backend= TRTLLM_RAGGED (MLAPrefillBackendEnum.TRTLLM_RAGGED). bench: native `vllm bench serve` runs under the container's system Python (the interpreter backing the vllm CLI), not the job .venv, so the pandas SpeedBench needs wasn't installed there — only the docker branch handled it. Add a native branch that installs pandas into that interpreter. AI-assisted change. Co-Authored-By: Claude --- lib/run_vllm_bench.sh | 18 ++++++++++++++++-- workloads/kimi_k2_5_b200.yaml | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index 3c7fe7d..ef72114 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -57,9 +57,10 @@ PY fi test -s "${data_dir}/${subset}.jsonl" - # Docker runtime: ship the data into the container and make sure pandas is - # available there (vLLM's SpeedBench loads the JSONL via pandas). + # vLLM's SpeedBench loads the JSONL via pandas, which the release image + # lacks. Ensure pandas is importable by whatever Python runs `vllm bench`. if [[ "$runtime" != "native" ]]; then + # Docker runtime: ship the data into the container and install there. docker exec "$container" mkdir -p "$data_dir" docker cp "${data_dir}/." "${container}:${data_dir}/" if ! docker exec "$container" python3 -c 'import pandas' 2>/dev/null; then @@ -68,6 +69,19 @@ PY 'PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --quiet pandas \ || PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --user --quiet pandas' fi + else + # Native runtime: `vllm bench serve` runs under the interpreter backing the + # `vllm` CLI (the container's system Python), not the job .venv, so pandas + # installed into the .venv above isn't visible to it. + local vllm_bin vllm_py + vllm_bin="$(command -v vllm || true)" + vllm_py="$(awk 'NR==1{sub(/^#![[:space:]]*/,""); print $1; exit}' "$vllm_bin" 2>/dev/null)" + [[ -x "$vllm_py" ]] || vllm_py="$(command -v python3)" + if ! "$vllm_py" -c 'import pandas' 2>/dev/null; then + echo "--- :python: installing pandas for native vllm bench ($vllm_py)" + PIP_BREAK_SYSTEM_PACKAGES=1 "$vllm_py" -m pip install --quiet pandas \ + || PIP_BREAK_SYSTEM_PACKAGES=1 "$vllm_py" -m pip install --user --quiet pandas + fi fi } diff --git a/workloads/kimi_k2_5_b200.yaml b/workloads/kimi_k2_5_b200.yaml index d04899f..f43efb7 100644 --- a/workloads/kimi_k2_5_b200.yaml +++ b/workloads/kimi_k2_5_b200.yaml @@ -11,7 +11,7 @@ vllm: --trust-remote-code --kv-cache-dtype fp8 --tensor-parallel-size 8 - --attention-config.use_trtllm_ragged_deepseek_prefill=True + --attention-config.mla_prefill_backend=TRTLLM_RAGGED --tool-call-parser kimi_k2 --enable-auto-tool-choice --reasoning-parser kimi_k2