diff --git a/workloads/nemotron_3_ultra_550b_b200.yaml b/workloads/nemotron_3_ultra_550b_b200.yaml new file mode 100644 index 0000000..b8a5a1f --- /dev/null +++ b/workloads/nemotron_3_ultra_550b_b200.yaml @@ -0,0 +1,50 @@ +# Nemotron-3-Ultra-550B-A55B (FP8) on H200 +# Hybrid Mamba-2 + LatentMoE architecture (550B total, 55B active). +name: nemotron_3_ultra-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 + serve_args: >- + --trust-remote-code + --kv-cache-dtype fp8 + --max-num-seqs 16 + --max-model-len 32768 + --gpu-memory-utilization 0.9 + --enable-flashinfer-autotune + --mamba-backend triton + --mamba-ssm-cache-dtype float32 + -cc.pass_config.fuse_allreduce_rms=False + --tensor-parallel-size 8 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --reasoning-parser nemotron_v3 + --speculative_config.method mtp + --speculative_config.num_speculative_tokens 5 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy \ No newline at end of file diff --git a/workloads/nemotron_3_ultra_550b_h200.yaml b/workloads/nemotron_3_ultra_550b_h200.yaml new file mode 100644 index 0000000..0c25760 --- /dev/null +++ b/workloads/nemotron_3_ultra_550b_h200.yaml @@ -0,0 +1,50 @@ +# Nemotron-3-Ultra-550B-A55B (FP8) on H200 +# Hybrid Mamba-2 + LatentMoE architecture (550B total, 55B active). +name: nemotron_3_ultra-h200 +gpu: H200 +num_gpus: 8 +nightly: true + +vllm: + model: RedHatAI/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8-dynamic + serve_args: >- + --trust-remote-code + --kv-cache-dtype fp8 + --max-num-seqs 16 + --max-model-len 32768 + --gpu-memory-utilization 0.9 + --enable-flashinfer-autotune + --mamba-backend triton + --mamba-ssm-cache-dtype float32 + -cc.pass_config.fuse_allreduce_rms=False + --tensor-parallel-size 8 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --reasoning-parser nemotron_v3 + --speculative_config.method mtp + --speculative_config.num_speculative_tokens 5 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy \ No newline at end of file