From 6203ddd4b89bc5929a2a7369d9fc0d043015f2f1 Mon Sep 17 00:00:00 2001 From: dougbtv Date: Tue, 19 May 2026 10:06:15 -0400 Subject: [PATCH 01/62] Add BFCL function-calling evaluation support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new `bfcl:` recipe block for running Berkeley Function Calling Leaderboard categories against a live vLLM server. Results are transformed to lm_eval-compatible JSON so the existing ingestion pipeline and dashboard auto-discover scores as `bfcl_` tasks — zero changes to ingest.py or the dashboard. - lib/parse_workload.py: parse/validate bfcl block, emit BFCL_TSV - lib/run_bfcl.py: register model, run generate+evaluate, transform output - lib/run.sh: dispatch loop for bfcl categories after lm_eval - .buildkite/generate_pipeline.py: conditional bfcl-eval install - workloads/deepseek_v3_2_h200.yaml: enable 4 BFCL categories - README.md: document bfcl block schema This PR was authored with assistance from Claude Code. Co-Authored-By: Claude Opus 4.6 Signed-off-by: dougbtv Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 8 +- README.md | 14 +- lib/parse_workload.py | 54 +++++++- lib/run.sh | 15 ++- lib/run_bfcl.py | 212 ++++++++++++++++++++++++++++++ workloads/deepseek_v3_2_h200.yaml | 8 ++ 6 files changed, 303 insertions(+), 8 deletions(-) create mode 100644 lib/run_bfcl.py diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index a709cc8..38ae03c 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -164,7 +164,13 @@ def make_step(path, data, profiles): bench_only = is_truthy(os.environ.get("BENCH_ONLY")) or is_truthy( data.get("bench_only") ) - setup_commands = BENCH_ONLY_SETUP_COMMANDS if bench_only else FULL_SETUP_COMMANDS + has_bfcl = bool(data.get("bfcl")) + if bench_only: + setup_commands = BENCH_ONLY_SETUP_COMMANDS + elif has_bfcl: + setup_commands = [setup_command("'lm-eval[api]' pyyaml bfcl-eval soundfile")] + else: + setup_commands = FULL_SETUP_COMMANDS step = { "label": f"{emoji} {name}", "agents": {"queue": queue}, diff --git a/README.md b/README.md index a1330e3..93bfb1b 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,14 @@ CLAUDE.md agent conventions and detailed Buildkite workflow ### Recipe schema -A recipe has top-level metadata plus three blocks: +A recipe has top-level metadata plus up to three eval blocks: - **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides). Required. - **`lm_eval:`** — *what accuracy to measure.* Lists lm-evaluation-harness tasks to run against the live server (e.g. `gsm8k`, `aime25`). Each task's score is saved under `results///`. Optional. - **`vllm_bench:`** — *what perf to measure.* Lists `vllm bench serve` configs (input/output lengths, concurrency, dataset). Raw JSON is saved and ingested into the perf dashboard. Optional. +- **`bfcl:`** — *function-calling eval.* Runs [BFCL](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) test categories against the live server. Some models need `--enable-auto-tool-choice` and `--tool-call-parser` in `serve_args`. Results are transformed to lm_eval format and ingested as `bfcl_` tasks. Optional. -Include either or both of `lm_eval:` / `vllm_bench:` depending on what you want out of this recipe. +Include one or more of `lm_eval:` / `vllm_bench:` / `bfcl:` depending on what you want out of this recipe. ```yaml name: qwen3_5-h200 # used in container name and results// @@ -60,6 +61,14 @@ lm_eval: # accuracy tasks (optional) - name: aime25 num_fewshot: 0 +bfcl: # function-calling eval (optional) + test_categories: # BFCL test categories to run + - simple_python + - multiple + - parallel + num_threads: 8 # optional, default 8 + temperature: 0.001 # optional, default 0.001 + vllm_bench: # perf runs (optional) — fed to the perf dashboard configs: - name: 1k-in-1k-out-conc-256 @@ -76,6 +85,7 @@ A few things worth knowing: - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass. +- **`bfcl` may need tool-call serve args.** Some models require `--enable-auto-tool-choice` and `--tool-call-parser` for function-calling; the parser warns if `--tool-call-parser` is absent. Each category runs as a separate generate + evaluate pass; scores appear on the eval dashboard as `bfcl_` tasks. For everything else (the full set of supported fields, defaults, validation rules), the existing files in `workloads/` are the working reference and `lib/parse_workload.py` is the source of truth. diff --git a/lib/parse_workload.py b/lib/parse_workload.py index ce9f0b7..91d6a5a 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -26,6 +26,18 @@ "speed_bench_dataset_subset", "speed_bench_category", } BENCH_REQUIRED = ("name", "input_len", "output_len", "num_prompts", "max_concurrency") +BFCL_FIELDS = {"test_categories", "num_threads", "temperature"} +BFCL_KNOWN_CATEGORIES = { + "simple_python", "simple_java", "simple_javascript", + "multiple", "parallel", "parallel_multiple", "irrelevance", + "live_simple", "live_multiple", "live_parallel", + "live_parallel_multiple", "live_irrelevance", "live_relevance", + "multi_turn_base", "multi_turn_miss_func", + "multi_turn_miss_param", "multi_turn_long_context", + "memory_kv", "memory_vector", "memory_rec_sum", + "all", "all_scoring", "single_turn", "multi_turn", + "live", "non_live", "non_python", "python", "memory", "agentic", +} COMMIT_IMAGE_TEMPLATE = "vllm/vllm-openai:nightly-{commit}" @@ -181,6 +193,28 @@ def opt(key): return "\n".join(lines) +def validate_bfcl(bfcl: dict, serve_args: str, path: str) -> None: + extra = set(bfcl) - BFCL_FIELDS + if extra: + sys.exit(f"{path}: bfcl block has unsupported fields {sorted(extra)}") + cats = bfcl.get("test_categories") or [] + if not cats: + sys.exit(f"{path}: bfcl block requires at least one test_categories entry") + for cat in cats: + if cat not in BFCL_KNOWN_CATEGORIES: + sys.exit(f"{path}: unknown bfcl test category {cat!r}") + if "--tool-call-parser" not in serve_args: + print(f"WARNING: {path}: bfcl without --tool-call-parser in serve_args; " + "some models may need it for function-calling", file=sys.stderr) + + +def bfcl_tsv(bfcl: dict) -> str: + cats = bfcl.get("test_categories") or [] + num_threads = bfcl.get("num_threads", 8) + temperature = bfcl.get("temperature", 0.001) + return "\n".join(f"{cat}\t{num_threads}\t{temperature}" for cat in cats) + + def main(path: str) -> None: with open(path) as f: data = yaml.safe_load(f) @@ -194,7 +228,18 @@ def main(path: str) -> None: bench = data.get("vllm_bench") or {} tasks = lm_eval.get("tasks") or [] - validate_tasks(tasks, path) + bfcl = data.get("bfcl") or {} + bench_configs = bench.get("configs") or [] + + if not tasks and not bench_configs and not bfcl: + sys.exit(f"{path}: workload must define at least one of lm_eval, vllm_bench, or bfcl") + + if tasks: + validate_tasks(tasks, path) + + serve_args = vllm.get("serve_args") or "" + if bfcl: + validate_bfcl(bfcl, serve_args, path) image, vllm_commit = resolve_image(vllm) env = {**(profile.get("env") or {}), **(vllm.get("env") or {})} @@ -204,17 +249,18 @@ def main(path: str) -> None: metadata = bench.get("metadata") or {} tp = metadata.get("tp") if tp is None: - tp = parse_tp(vllm.get("serve_args") or "") + tp = parse_tp(serve_args) emit("NAME", data.get("name", "")) emit("IMAGE", image) emit("VLLM_COMMIT", vllm_commit) emit("MODEL", vllm.get("model", "")) - emit("SERVE_ARGS", vllm.get("serve_args", "")) + emit("SERVE_ARGS", serve_args) emit("SERVER_RUNTIME", profile.get("server_runtime", "docker")) emit("ENV", "\n".join(f"{k}={fmt(v)}" for k, v in env.items())) emit("LM_EVAL_TASKS_TSV", task_tsv(tasks, lm_eval.get("model_args") or {})) - emit("VLLM_BENCH_TSV", bench_tsv(bench.get("configs") or [], path)) + emit("VLLM_BENCH_TSV", bench_tsv(bench_configs, path)) + emit("BFCL_TSV", bfcl_tsv(bfcl) if bfcl else "") emit("BENCH_DEVICE", metadata.get("device") or gpu.lower()) emit("BENCH_TP", tp) emit("BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or "")) diff --git a/lib/run.sh b/lib/run.sh index 005c1a2..83a71e2 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -58,7 +58,7 @@ while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset done <<< "$WORKLOAD_VLLM_BENCH_TSV" if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then - echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval tasks" + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" exit 0 fi @@ -73,3 +73,16 @@ while IFS=$'\t' read -r task fewshot model_args; do --task "$task" \ ${INGEST_NO_SAMPLES:+--no-samples} || true done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + +# bfcl function-calling eval +while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true +done <<< "$WORKLOAD_BFCL_TSV" diff --git a/lib/run_bfcl.py b/lib/run_bfcl.py new file mode 100644 index 0000000..325c176 --- /dev/null +++ b/lib/run_bfcl.py @@ -0,0 +1,212 @@ +"""Run a single BFCL test category against a running vLLM server. + +Usage: + python3 lib/run_bfcl.py \ + + +Registers the model in BFCL's config, runs generate + evaluate, then +writes an lm_eval-compatible results JSON so the existing ingest.py +and dashboard auto-discover the scores without any adapter. +""" + +import csv +import json +import os +import sys +import time +from pathlib import Path +from urllib.parse import urlparse + + +def get_typer_defaults(func): + """Extract default kwargs from a Typer-decorated function.""" + import typer + + defaults = {} + for name, default in zip( + func.__annotations__.keys(), + func.__defaults__, + strict=True, + ): + if isinstance(default, typer.models.OptionInfo): + defaults[name] = default.default + return defaults + + +def register_model(model: str, base_url: str): + """Inject the model into BFCL's MODEL_CONFIG_MAPPING.""" + import bfcl_eval.constants.model_config as bfcl_model_config + from bfcl_eval.model_handler.api_inference.openai_completion import ( + OpenAICompletionsHandler, + ) + + bfcl_model_config.MODEL_CONFIG_MAPPING = { + model: bfcl_model_config.ModelConfig( + model_name=model, + display_name=f"{model} (FC) (vLLM)", + url=f"https://huggingface.co/{model}", + org="", + license="apache-2.0", + model_handler=OpenAICompletionsHandler, + input_price=None, + output_price=None, + is_fc_model=True, + underscore_to_dot=True, + ) + } + + +def run_generate(model, category, num_threads, temperature): + from bfcl_eval.__main__ import generate + + kwargs = get_typer_defaults(generate) + kwargs["model"] = [model] + kwargs["test_category"] = category + kwargs["skip_server_setup"] = True + kwargs["num_threads"] = num_threads + kwargs["temperature"] = temperature + generate(**kwargs) + + +def run_evaluate(model, category): + from bfcl_eval.__main__ import evaluate + + kwargs = get_typer_defaults(evaluate) + kwargs["model"] = [model] + kwargs["test_category"] = category + evaluate(**kwargs) + + +CATEGORY_TO_CSV = { + "simple_python": "data_non_live.csv", + "simple_java": "data_non_live.csv", + "simple_javascript": "data_non_live.csv", + "multiple": "data_non_live.csv", + "parallel": "data_non_live.csv", + "parallel_multiple": "data_non_live.csv", + "irrelevance": "data_non_live.csv", + "live_simple": "data_live.csv", + "live_multiple": "data_live.csv", + "live_parallel": "data_live.csv", + "live_parallel_multiple": "data_live.csv", + "live_irrelevance": "data_live.csv", + "live_relevance": "data_live.csv", + "multi_turn_base": "data_multi_turn.csv", + "multi_turn_miss_func": "data_multi_turn.csv", + "multi_turn_miss_param": "data_multi_turn.csv", + "multi_turn_long_context": "data_multi_turn.csv", +} + + +def parse_score_from_csv(work_dir: Path, model: str, category: str) -> dict | None: + """Extract per-category accuracy from BFCL V4 aggregate CSV files.""" + csv_name = CATEGORY_TO_CSV.get(category) + csv_candidates = [work_dir / "score" / csv_name] if csv_name else [] + csv_candidates.append(work_dir / "score" / "data_overall.csv") + + bfcl_category = f"BFCL_v4_{category}" + for csv_path in csv_candidates: + if not csv_path.exists(): + continue + with open(csv_path) as f: + reader = csv.DictReader(f) + for row in reader: + model_name = row.get("Model", row.get("model", "")) + if model not in model_name and model.replace("/", "_") not in model_name: + continue + for key, value in row.items(): + if bfcl_category in key: + try: + return {"accuracy": float(value) / 100.0} + except (ValueError, TypeError): + continue + + # Fallback: look for per-category JSONL score files (older BFCL versions) + model_slug = model.replace("/", "_") + for p in (work_dir / "score").rglob(f"{category}_score.json"): + with open(p) as f: + return json.loads(f.readline()) + + return None + + +def to_lm_eval_format(model: str, category: str, score: dict) -> dict: + """Transform BFCL aggregate score into lm_eval-compatible results JSON.""" + task_name = f"bfcl_{category}" + accuracy = score.get("accuracy", 0.0) + + return { + "results": { + task_name: { + "acc,none": accuracy, + "acc_stderr,none": 0.0, + "alias": task_name, + } + }, + "configs": { + task_name: {"task": task_name, "num_fewshot": 0} + }, + "versions": {task_name: 1}, + "n-shot": {task_name: 0}, + "config": { + "model": "local-completions", + "model_args": f"model={model}", + }, + } + + +def main(): + if len(sys.argv) != 7: + sys.exit( + "usage: run_bfcl.py " + " " + ) + + model, base_url, category = sys.argv[1], sys.argv[2], sys.argv[3] + num_threads, temperature = int(sys.argv[4]), float(sys.argv[5]) + results_dir = Path(sys.argv[6]) + + work_dir = (results_dir / ".bfcl_work").resolve() + work_dir.mkdir(parents=True, exist_ok=True) + + parsed = urlparse(base_url) + host = parsed.hostname or "localhost" + port = parsed.port or 8000 + + os.environ["OPENAI_BASE_URL"] = base_url + os.environ["OPENAI_API_KEY"] = os.environ.get("OPENAI_API_KEY", "dummy") + os.environ["BFCL_PROJECT_ROOT"] = str(work_dir) + os.environ["LOCAL_SERVER_ENDPOINT"] = f"http://{host}" + os.environ["LOCAL_SERVER_PORT"] = str(port) + + register_model(model, base_url) + + print(f"[bfcl] generate: model={model} category={category}", flush=True) + run_generate(model, category, num_threads, temperature) + + print(f"[bfcl] evaluate: model={model} category={category}", flush=True) + run_evaluate(model, category) + + score = parse_score_from_csv(work_dir, model, category) + if not score: + sys.exit(f"[bfcl] no score found for {category}") + + print( + f"[bfcl] {category}: accuracy={score.get('accuracy', '?')}", + flush=True, + ) + + out_dir = results_dir / f"bfcl-{category}" + out_dir.mkdir(parents=True, exist_ok=True) + ts = time.strftime("%Y-%m-%dT%H-%M-%S") + out_path = out_dir / f"results_{ts}.json" + + lm_eval_results = to_lm_eval_format(model, category, score) + with open(out_path, "w") as f: + json.dump(lm_eval_results, f, indent=2) + + print(f"[bfcl] results written to {out_path}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/workloads/deepseek_v3_2_h200.yaml b/workloads/deepseek_v3_2_h200.yaml index 49656a0..3731435 100644 --- a/workloads/deepseek_v3_2_h200.yaml +++ b/workloads/deepseek_v3_2_h200.yaml @@ -34,6 +34,14 @@ lm_eval: max_length: 32768 max_gen_toks: 4096 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + vllm_bench: configs: - name: 8k-in-1k-out-conc-128 From c539b031753e7b70da8aaa728e3481657bb57554 Mon Sep 17 00:00:00 2001 From: dougbtv Date: Tue, 19 May 2026 13:55:18 -0400 Subject: [PATCH 02/62] fix(bfcl): add /v1 prefix to OpenAI base URL and fix score file glob BFCL's OpenAI client was hitting /chat/completions instead of /v1/chat/completions, causing 404s from vLLM. Also fix the JSONL fallback glob to match BFCL V4 filenames (BFCL_v4__score.json). Co-Authored-By: Claude Signed-off-by: dougbtv Signed-off-by: Stacy Roberts --- lib/run_bfcl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/run_bfcl.py b/lib/run_bfcl.py index 325c176..545244e 100644 --- a/lib/run_bfcl.py +++ b/lib/run_bfcl.py @@ -123,7 +123,7 @@ def parse_score_from_csv(work_dir: Path, model: str, category: str) -> dict | No # Fallback: look for per-category JSONL score files (older BFCL versions) model_slug = model.replace("/", "_") - for p in (work_dir / "score").rglob(f"{category}_score.json"): + for p in (work_dir / "score").rglob(f"*{category}_score.json"): with open(p) as f: return json.loads(f.readline()) @@ -173,7 +173,7 @@ def main(): host = parsed.hostname or "localhost" port = parsed.port or 8000 - os.environ["OPENAI_BASE_URL"] = base_url + os.environ["OPENAI_BASE_URL"] = base_url.rstrip("/") + "/v1" os.environ["OPENAI_API_KEY"] = os.environ.get("OPENAI_API_KEY", "dummy") os.environ["BFCL_PROJECT_ROOT"] = str(work_dir) os.environ["LOCAL_SERVER_ENDPOINT"] = f"http://{host}" From 85bd1ad7f906bf5f8d3c0140c2fdeab00b73a94d Mon Sep 17 00:00:00 2001 From: dougbtv Date: Tue, 19 May 2026 14:39:33 -0400 Subject: [PATCH 03/62] add Buildkite log group headers for BFCL categories BFCL output was hidden inside the last lm_eval collapsed group. Each category now gets its own `--- :phone: bfcl ` header. Co-Authored-By: Claude Signed-off-by: dougbtv Signed-off-by: Stacy Roberts --- lib/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/run.sh b/lib/run.sh index 83a71e2..96ce957 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -77,6 +77,7 @@ done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" # bfcl function-calling eval while IFS=$'\t' read -r category num_threads temperature; do [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ "$category" "$num_threads" "$temperature" "$RESULTS_DIR" From 989bc28d5e5dc4bea0038b3f6732c2524f7c67df Mon Sep 17 00:00:00 2001 From: khluu Date: Tue, 19 May 2026 13:34:18 -0700 Subject: [PATCH 04/62] Add DeepSeek-V4-Flash GSM8K workload on B200 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TP=2 × DP=4 + EP with deep_gemm_mega_moe backend and MTP speculative decoding, matching vllm/vllm#42111. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Stacy Roberts --- workloads/deepseek_v4_flash_b200.yaml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 workloads/deepseek_v4_flash_b200.yaml diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml new file mode 100644 index 0000000..6cebf23 --- /dev/null +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -0,0 +1,49 @@ +# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe, MTP spec-decode) +name: deepseek_v4_flash-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: deepseek-ai/DeepSeek-V4-Flash + serve_args: >- + --tensor-parallel-size 2 + --data-parallel-size 4 + --enable-expert-parallel + --max-model-len 32768 + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm_mega_moe + --attention_config.use_fp4_indexer_cache=True + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --trust-remote-code + --speculative_config.method=mtp + --speculative_config.num_speculative_tokens=2 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From ac8ccfd6ad66735228398a142525b13534440996 Mon Sep 17 00:00:00 2001 From: khluu Date: Wed, 20 May 2026 02:48:32 -0700 Subject: [PATCH 05/62] Remove vllm_bench from DSv4-Flash workload The speed_bench dataset depends on an external URL (opencompass.openxlab.space) that's unreachable from the B200 k8s pods, causing the entire run to fail before lm_eval runs. Drop vllm_bench for now; can re-add once the dataset is cached. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Stacy Roberts --- workloads/deepseek_v4_flash_b200.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index 6cebf23..e57c643 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -35,15 +35,3 @@ lm_eval: num_concurrent: 64 max_length: 32768 max_gen_toks: 8192 - -vllm_bench: - configs: - - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench - input_len: 8192 - output_len: 1024 - num_prompts: 512 - max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy From 5cf79f62befb952da11db9b05ec79ceece0698de Mon Sep 17 00:00:00 2001 From: dougbtv Date: Tue, 26 May 2026 13:01:23 -0400 Subject: [PATCH 06/62] Add gpt-oss-120b H200 workload for nightly perf & eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commercially critical model with an active perf regression (vllm#40838, ~16% throughput drop v0.16→v0.19). Adding nightly tracking so future regressions are caught at the commit that introduces them. Config: 8xH200 TP=8, gsm8k 5-shot eval, 8k-in/1k-out throughput bench. Signed-off-by: dougbtv Co-Authored-By: Claude Signed-off-by: Stacy Roberts --- workloads/gpt_oss_120b_h200.yaml | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 workloads/gpt_oss_120b_h200.yaml diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml new file mode 100644 index 0000000..476e4d6 --- /dev/null +++ b/workloads/gpt_oss_120b_h200.yaml @@ -0,0 +1,38 @@ +# GPT-OSS 120B on H200 (8xH200, TP=8) +# Tracking perf regression: https://github.com/vllm-project/vllm/issues/40838 +name: gpt_oss_120b-h200 +gpu: H200 +num_gpus: 8 +nightly: true + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 8 + --max-model-len 32768 + --max-num-seqs 512 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From bcb966731b9038d1a05b1c7650b381fae5d33aa1 Mon Sep 17 00:00:00 2001 From: dougbtv Date: Tue, 26 May 2026 13:16:29 -0400 Subject: [PATCH 07/62] Add BFCL tool-calling eval to gpt-oss-120b recipe Adds simple_python, multiple, parallel, parallel_multiple categories with --tool-call-parser openai and --enable-auto-tool-choice serve args. Signed-off-by: dougbtv Co-Authored-By: Claude Signed-off-by: Stacy Roberts --- workloads/gpt_oss_120b_h200.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index 476e4d6..e3af796 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -11,6 +11,8 @@ vllm: --tensor-parallel-size 8 --max-model-len 32768 --max-num-seqs 512 + --tool-call-parser openai + --enable-auto-tool-choice lm_eval: model_args: @@ -25,6 +27,14 @@ lm_eval: max_length: 32768 max_gen_toks: 8192 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + vllm_bench: configs: - name: 8k-in-1k-out-conc-128 From 868e0c7d1c18d5e9fe89dc7633a7317afa8b9ea3 Mon Sep 17 00:00:00 2001 From: dougbtv Date: Wed, 27 May 2026 14:56:00 -0400 Subject: [PATCH 08/62] Switch gpt-oss-120b to TP=4 Matches the regression benchmark config from vllm#40838 (TP=4 on H200). Signed-off-by: dougbtv Co-Authored-By: Claude Signed-off-by: Stacy Roberts --- workloads/gpt_oss_120b_h200.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index e3af796..03975f3 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -1,4 +1,4 @@ -# GPT-OSS 120B on H200 (8xH200, TP=8) +# GPT-OSS 120B on H200 (8xH200, TP=4) # Tracking perf regression: https://github.com/vllm-project/vllm/issues/40838 name: gpt_oss_120b-h200 gpu: H200 @@ -8,7 +8,7 @@ nightly: true vllm: model: openai/gpt-oss-120b serve_args: >- - --tensor-parallel-size 8 + --tensor-parallel-size 4 --max-model-len 32768 --max-num-seqs 512 --tool-call-parser openai From fcfe364dc6e65624151bb6989bc155497e35205d Mon Sep 17 00:00:00 2001 From: dougbtv Date: Wed, 27 May 2026 15:04:34 -0400 Subject: [PATCH 09/62] Add BFCL tool-calling eval to all tool-calling-capable models Adds simple_python, multiple, parallel, parallel_multiple BFCL categories to DSv4 Pro, DSv4 Flash, GLM 5.1, Kimi K2.5, and MiniMax M2.5. All already had --tool-call-parser and --enable-auto-tool-choice in their serve_args. Signed-off-by: dougbtv Co-Authored-By: Claude Signed-off-by: Stacy Roberts --- workloads/deepseek_v4_flash_b200.yaml | 8 ++++++++ workloads/deepseek_v4_pro_5_h200.yaml | 8 ++++++++ workloads/glm_5_1_h200.yaml | 8 ++++++++ workloads/kimi_k2_5_h200.yaml | 8 ++++++++ workloads/minimax_m2_5_h200.yaml | 8 ++++++++ 5 files changed, 40 insertions(+) diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index e57c643..694ab37 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -23,6 +23,14 @@ vllm: --speculative_config.method=mtp --speculative_config.num_speculative_tokens=2 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + lm_eval: model_args: tokenized_requests: false diff --git a/workloads/deepseek_v4_pro_5_h200.yaml b/workloads/deepseek_v4_pro_5_h200.yaml index 6ec978c..27ef33d 100644 --- a/workloads/deepseek_v4_pro_5_h200.yaml +++ b/workloads/deepseek_v4_pro_5_h200.yaml @@ -39,6 +39,14 @@ lm_eval: max_length: 32768 max_gen_toks: 8192 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + vllm_bench: configs: - name: 8k-in-1k-out-conc-128 diff --git a/workloads/glm_5_1_h200.yaml b/workloads/glm_5_1_h200.yaml index fa2a9e8..90bacd8 100644 --- a/workloads/glm_5_1_h200.yaml +++ b/workloads/glm_5_1_h200.yaml @@ -30,6 +30,14 @@ lm_eval: max_length: 40960 max_gen_toks: 32768 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + vllm_bench: configs: - name: 8k-in-1k-out-conc-128 diff --git a/workloads/kimi_k2_5_h200.yaml b/workloads/kimi_k2_5_h200.yaml index 0eac34a..684bcca 100644 --- a/workloads/kimi_k2_5_h200.yaml +++ b/workloads/kimi_k2_5_h200.yaml @@ -27,6 +27,14 @@ lm_eval: max_length: 40960 max_gen_toks: 32768 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + vllm_bench: configs: - name: 8k-in-1k-out-conc-128 diff --git a/workloads/minimax_m2_5_h200.yaml b/workloads/minimax_m2_5_h200.yaml index 4c13838..58f7349 100644 --- a/workloads/minimax_m2_5_h200.yaml +++ b/workloads/minimax_m2_5_h200.yaml @@ -30,6 +30,14 @@ lm_eval: max_length: 40960 max_gen_toks: 32768 +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + vllm_bench: configs: - name: 8k-in-1k-out-conc-128 From baec0620336265e235074d058897a8e9daa436f6 Mon Sep 17 00:00:00 2001 From: khluu Date: Thu, 28 May 2026 14:36:43 -0700 Subject: [PATCH 10/62] Use ECR pull-through cache for B200 K8s pod images B200 jobs pulling from public.ecr.aws hit 429 rate limits, causing ImagePullBackOff failures. Rewrite public ECR URLs to the private pull-through cache (same pattern CI already uses for K8s jobs). Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 38ae03c..82d3703 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -59,6 +59,18 @@ def setup_command(packages): "A100": ":a100:", } +ECR_PUBLIC_PREFIX = "public.ecr.aws/" +ECR_PULL_THROUGH_CACHE = ( + "936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/" +) + + +def ecr_pull_through(image): + """Rewrite public ECR URLs to the private pull-through cache.""" + if image.startswith(ECR_PUBLIC_PREFIX): + return ECR_PULL_THROUGH_CACHE + image[len(ECR_PUBLIC_PREFIX):] + return image + def is_truthy(value): return str(value or "").lower() in {"1", "true", "yes"} @@ -179,7 +191,7 @@ def make_step(path, data, profiles): "artifact_paths": ["results/**/*"], } if profile.get("server_runtime") == "native": - step["plugins"] = [b200_k8s_plugin(resolved_image(data), data.get("num_gpus", 1))] + step["plugins"] = [b200_k8s_plugin(ecr_pull_through(resolved_image(data)), data.get("num_gpus", 1))] step_env = { k: os.environ[k] for k in ("VLLM_IMAGE", "VLLM_COMMIT", "BENCH_ONLY") From 3ac87c4c4865f75caebf586dbf3c4d775599bb19 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 29 May 2026 03:44:03 -0700 Subject: [PATCH 11/62] Delete workloads/deepseek_v3_2_h200.yaml Signed-off-by: Kevin H. Luu Signed-off-by: Stacy Roberts --- workloads/deepseek_v3_2_h200.yaml | 55 ------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 workloads/deepseek_v3_2_h200.yaml diff --git a/workloads/deepseek_v3_2_h200.yaml b/workloads/deepseek_v3_2_h200.yaml deleted file mode 100644 index 3731435..0000000 --- a/workloads/deepseek_v3_2_h200.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# DeepSeek-V3.2 on H200 -# Recipe: TP=2 preferred on Hopper (TP=8 hits FlashMLA-Sparse head-padding overhead). -# With 8 GPUs we run TP=2 x DP=4 + EP. -name: deepseek_v3_2-h200 -gpu: H200 -num_gpus: 8 -nightly: true - -vllm: - model: deepseek-ai/DeepSeek-V3.2 - env: - TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas - serve_args: >- - --tensor-parallel-size 2 - --data-parallel-size 4 - --enable-expert-parallel - --max-model-len 32768 - --tokenizer-mode deepseek_v32 - --tool-call-parser deepseek_v32 - --reasoning-parser deepseek_v3 - --enable-auto-tool-choice - --trust-remote-code - -lm_eval: - model_args: - tokenized_requests: false - tokenizer_backend: null - timeout: 6000 - tasks: - - name: gsm8k - num_fewshot: 5 - model_args: - num_concurrent: 64 - max_length: 32768 - max_gen_toks: 4096 - -bfcl: - test_categories: - - simple_python - - multiple - - parallel - - parallel_multiple - num_threads: 8 - -vllm_bench: - configs: - - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench - input_len: 8192 - output_len: 1024 - num_prompts: 512 - max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy From b40984a861a7a7dfb40036fa546977bccfbd547d Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Mon, 1 Jun 2026 20:59:06 +0000 Subject: [PATCH 12/62] add mi355 and kimi k2.5 coverage Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 1 + README.md | 1 + lib/gpu_profiles.yaml | 7 +++++ lib/parse_workload.py | 21 +++++++++----- lib/server.sh | 9 +++++- workloads/kimi_k2_5_mi355x.yaml | 50 +++++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 workloads/kimi_k2_5_mi355x.yaml diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 82d3703..aad9638 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -57,6 +57,7 @@ def setup_command(packages): "H200": ":h200:", "B200": ":b200:", "A100": ":a100:", + "MI355X": ":amd:", } ECR_PUBLIC_PREFIX = "public.ecr.aws/" diff --git a/README.md b/README.md index 93bfb1b..0d05452 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. +- **AMD hardware** (e.g. the `MI355X` profile) uses two extra profile fields. `image_repo: vllm/vllm-openai-rocm` makes image resolution map `VLLM_COMMIT` onto that repo's `nightly-` tag instead of using the CUDA `VLLM_IMAGE` verbatim (an explicit ROCm `VLLM_IMAGE` is still honored). `server_runtime: docker-rocm` makes the Docker runtime expose GPUs via `/dev/kfd` + `/dev/dri` (plus `--group-add video` and `--security-opt seccomp=unconfined`) instead of NVIDIA's `--gpus all`. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass. diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 329080c..6e691fa 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -13,3 +13,10 @@ B200: env: VLLM_DEEP_GEMM_WARMUP: skip NCCL_CUMEM_HOST_ENABLE: 0 + +MI355X: + queue: amd_mi355_8 + # Persistent HF cache mount is not yet set up on mi355 agents + hf_home: /mnt/shared/hf-models + image_repo: vllm/vllm-openai-rocm + server_runtime: docker-rocm diff --git a/lib/parse_workload.py b/lib/parse_workload.py index 91d6a5a..a6e7bbc 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -38,7 +38,6 @@ "all", "all_scoring", "single_turn", "multi_turn", "live", "non_live", "non_python", "python", "memory", "agentic", } -COMMIT_IMAGE_TEMPLATE = "vllm/vllm-openai:nightly-{commit}" def emit(name: str, value: object) -> None: @@ -89,15 +88,23 @@ def load_profile(gpu: str, workload_path: str) -> dict: return profiles[gpu] -def resolve_image(vllm: dict) -> tuple[str, str]: +def resolve_image(vllm: dict, profile: dict) -> tuple[str, str]: """Pick the image and commit using VLLM_IMAGE / VLLM_COMMIT / workload.""" override_image = (os.environ.get("VLLM_IMAGE") or "").strip() override_commit = (os.environ.get("VLLM_COMMIT") or "").strip() - if override_image: + # ROCm images are located at vllm/vllm-openai-rocm. The default + # images (CUDA) are stored at vllm/vllm-openai + custom_repo = (profile.get("image_repo") or "").strip() + repo = custom_repo or "vllm/vllm-openai" + + if override_image and (not custom_repo or repo in override_image): return override_image, override_commit or commit_from_image(override_image) - if override_commit: - return COMMIT_IMAGE_TEMPLATE.format(commit=override_commit), override_commit - image = vllm.get("image", "vllm/vllm-openai:nightly") + + commit = override_commit or commit_from_image(override_image) + if commit: + return f"{repo}:nightly-{commit}", commit + + image = vllm.get("image", f"{repo}:nightly") return image, commit_from_image(str(image)) @@ -241,7 +248,7 @@ def main(path: str) -> None: if bfcl: validate_bfcl(bfcl, serve_args, path) - image, vllm_commit = resolve_image(vllm) + image, vllm_commit = resolve_image(vllm, profile) env = {**(profile.get("env") or {}), **(vllm.get("env") or {})} if "HF_HOME" not in env and profile.get("hf_home"): env["HF_HOME"] = profile["hf_home"] diff --git a/lib/server.sh b/lib/server.sh index c80fdbf..d1368e1 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -35,7 +35,14 @@ start_server() { return fi - local docker_args=(--gpus all --ipc=host --ulimit nofile=65536:65536 + local gpu_args + if [[ "$runtime" == "docker-rocm" ]]; then + gpu_args=(--device=/dev/kfd --device=/dev/dri --group-add video + --cap-add SYS_PTRACE --security-opt seccomp=unconfined) + else + gpu_args=(--gpus all) + fi + local docker_args=("${gpu_args[@]}" --ipc=host --ulimit nofile=65536:65536 -e VLLM_ENGINE_READY_TIMEOUT_S=3600 -p "${port}:${port}") local hf_home="" diff --git a/workloads/kimi_k2_5_mi355x.yaml b/workloads/kimi_k2_5_mi355x.yaml new file mode 100644 index 0000000..7179a0b --- /dev/null +++ b/workloads/kimi_k2_5_mi355x.yaml @@ -0,0 +1,50 @@ +# Kimi-K2.5 on AMD MI355X +name: kimi_k2_5-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: trigger explicitly with WORKLOADS=kimi_k2_5_mi355x until the +# MI355X path is proven, then flip to nightly: true to join the schedule. +nightly: false + +vllm: + model: moonshotai/Kimi-K2.5 + serve_args: >- + --tensor-parallel-size 8 + --mm-encoder-tp-mode data + --tool-call-parser kimi_k2 + --reasoning-parser kimi_k2 + --enable-auto-tool-choice + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From 7f32755e68c8e7d1393b773bcbdde1ddcbc37c09 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Mon, 1 Jun 2026 21:10:03 +0000 Subject: [PATCH 13/62] enable aiter Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 2 +- workloads/kimi_k2_5_mi355x.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index aad9638..c7d7a47 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -57,7 +57,7 @@ def setup_command(packages): "H200": ":h200:", "B200": ":b200:", "A100": ":a100:", - "MI355X": ":amd:", + "MI355X": ":mi355:", } ECR_PUBLIC_PREFIX = "public.ecr.aws/" diff --git a/workloads/kimi_k2_5_mi355x.yaml b/workloads/kimi_k2_5_mi355x.yaml index 7179a0b..22fb2aa 100644 --- a/workloads/kimi_k2_5_mi355x.yaml +++ b/workloads/kimi_k2_5_mi355x.yaml @@ -8,6 +8,8 @@ nightly: false vllm: model: moonshotai/Kimi-K2.5 + env: + VLLM_ROCM_USE_AITER: 1 serve_args: >- --tensor-parallel-size 8 --mm-encoder-tp-mode data From dd515283806d7692fab4f46ffa9c970288efd430 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Wed, 3 Jun 2026 17:52:11 +0000 Subject: [PATCH 14/62] use amd GPU emoji Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 2 +- README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index c7d7a47..aad9638 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -57,7 +57,7 @@ def setup_command(packages): "H200": ":h200:", "B200": ":b200:", "A100": ":a100:", - "MI355X": ":mi355:", + "MI355X": ":amd:", } ECR_PUBLIC_PREFIX = "public.ecr.aws/" diff --git a/README.md b/README.md index 0d05452..93bfb1b 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. -- **AMD hardware** (e.g. the `MI355X` profile) uses two extra profile fields. `image_repo: vllm/vllm-openai-rocm` makes image resolution map `VLLM_COMMIT` onto that repo's `nightly-` tag instead of using the CUDA `VLLM_IMAGE` verbatim (an explicit ROCm `VLLM_IMAGE` is still honored). `server_runtime: docker-rocm` makes the Docker runtime expose GPUs via `/dev/kfd` + `/dev/dri` (plus `--group-add video` and `--security-opt seccomp=unconfined`) instead of NVIDIA's `--gpus all`. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass. From 492e4a52440759e8aba5f6d3f858509e087965c3 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Thu, 4 Jun 2026 15:36:33 +0000 Subject: [PATCH 15/62] create k8 plugin for mi355 Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 93 +++++++++++++++++++++++++++++---- lib/gpu_profiles.yaml | 6 +-- lib/parse_workload.py | 15 +++--- lib/server.sh | 9 +--- 4 files changed, 95 insertions(+), 28 deletions(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index aad9638..036e953 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -51,7 +51,7 @@ def setup_command(packages): DEFAULT_TIMEOUT = 120 PROFILES_PATH = os.path.join(os.path.dirname(__file__), "..", "lib", "gpu_profiles.yaml") -COMMIT_IMAGE_TEMPLATE = "vllm/vllm-openai:nightly-{commit}" +DEFAULT_IMAGE_REPO = "vllm/vllm-openai" GPU_EMOJI = { "H200": ":h200:", @@ -77,18 +77,37 @@ def is_truthy(value): return str(value or "").lower() in {"1", "true", "yes"} -def resolved_image(data): +def commit_from_image(image): + """Extract the commit SHA from a 'repo:nightly-' image tag.""" + ref = image.split("@", 1)[0] + tag = ref.rpartition(":")[2] + prefix = "nightly-" + if not tag.startswith(prefix): + return "" + # tag is "nightly-"; drop any trailing "-suffix" (e.g. -cu124). + return tag[len(prefix):].split("-", 1)[0] + + +def resolved_image(data, profile): + """Resolve the docker image for the k8s pod (mirrors parse_workload). + + Honors a profile ``image_repo`` (e.g. ROCm) the same way the in-job parser + does, so the pod pulls the right repo for the hardware. + """ vllm = data.get("vllm") or {} override_image = (os.environ.get("VLLM_IMAGE") or "").strip() - if override_image: - return override_image override_commit = (os.environ.get("VLLM_COMMIT") or "").strip() - if override_commit: - return COMMIT_IMAGE_TEMPLATE.format(commit=override_commit) - return vllm.get("image", "vllm/vllm-openai:latest") + custom_repo = (profile.get("image_repo") or "").strip() + repo = custom_repo or DEFAULT_IMAGE_REPO + if override_image and (not custom_repo or repo in override_image): + return override_image + commit = override_commit or commit_from_image(override_image) + if commit: + return f"{repo}:nightly-{commit}" + return vllm.get("image", f"{repo}:nightly") -def b200_k8s_plugin(image, num_gpus): +def b200_k8s_plugin(image, num_gpus, profile=None): return { "kubernetes": { "podSpec": { @@ -144,6 +163,57 @@ def b200_k8s_plugin(image, num_gpus): } +def mi355x_k8s_plugin(image, num_gpus, profile=None): + profile = profile or {} + hf_home = profile.get("hf_home") or "/root/.cache/huggingface" + return { + "kubernetes": { + "podSpec": { + "containers": [ + { + "image": image, + "resources": {"limits": {"amd.com/gpu": num_gpus}}, + "securityContext": { + "seccompProfile": {"type": "Unconfined"}, + "capabilities": {"add": ["IPC_LOCK", "SYS_PTRACE"]}, + }, + "volumeMounts": [ + {"name": "devshm", "mountPath": "/dev/shm"}, + {"name": "hf-cache", "mountPath": hf_home}, + ], + "env": [ + {"name": "VLLM_USAGE_SOURCE", "value": "ci-test"}, + {"name": "HF_HOME", "value": hf_home}, + { + "name": "HF_TOKEN", + "valueFrom": { + "secretKeyRef": { + "name": "hf-token-secret", + "key": "token", + }, + }, + }, + ], + }, + ], + "volumes": [ + {"name": "devshm", "emptyDir": {"medium": "Memory"}}, + { + "name": "hf-cache", + "hostPath": {"path": hf_home, "type": "DirectoryOrCreate"}, + }, + ], + }, + }, + } + + +K8S_PLUGINS = { + "nvidia": b200_k8s_plugin, + "amd": mi355x_k8s_plugin, +} + + def load_profiles(): with open(PROFILES_PATH) as f: return yaml.safe_load(f) @@ -192,7 +262,12 @@ def make_step(path, data, profiles): "artifact_paths": ["results/**/*"], } if profile.get("server_runtime") == "native": - step["plugins"] = [b200_k8s_plugin(ecr_pull_through(resolved_image(data)), data.get("num_gpus", 1))] + kind = profile.get("k8s_plugin", "nvidia") + builder = K8S_PLUGINS.get(kind) + if builder is None: + sys.exit(f"{path}: unknown k8s_plugin {kind!r} (have {', '.join(K8S_PLUGINS)})") + image = ecr_pull_through(resolved_image(data, profile)) + step["plugins"] = [builder(image, data.get("num_gpus", 1), profile)] step_env = { k: os.environ[k] for k in ("VLLM_IMAGE", "VLLM_COMMIT", "BENCH_ONLY") diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 6e691fa..65de5d9 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -16,7 +16,7 @@ B200: MI355X: queue: amd_mi355_8 - # Persistent HF cache mount is not yet set up on mi355 agents - hf_home: /mnt/shared/hf-models image_repo: vllm/vllm-openai-rocm - server_runtime: docker-rocm + server_runtime: native + k8s_plugin: amd + hf_home: /mnt/shared/hf-models diff --git a/lib/parse_workload.py b/lib/parse_workload.py index a6e7bbc..ff119be 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -13,7 +13,6 @@ """ import os -import re import shlex import sys @@ -60,14 +59,14 @@ def env_truthy(name: str) -> bool: def commit_from_image(image: str) -> str: - """Extract a commit SHA from an image tag, if one is embedded.""" - _, sep, tag = image.rpartition(":") - if not sep: + """Extract the commit SHA from a 'repo:nightly-' image tag.""" + ref = image.split("@", 1)[0] # drop any "@sha256:..." digest first + tag = ref.rpartition(":")[2] # text after the tag separator + prefix = "nightly-" + if not tag.startswith(prefix): return "" - tag = tag.split("@", 1)[0] - m = (re.match(r"nightly-([0-9a-f]{7,40})(?:[-_.].*)?$", tag, re.IGNORECASE) - or re.search(r"(?:^|[-_.])([0-9a-f]{12,40})(?:$|[-_.])", tag, re.IGNORECASE)) - return m.group(1) if m else "" + # tag is "nightly-"; drop any trailing "-suffix" (e.g. -cu124). + return tag[len(prefix):].split("-", 1)[0] def known_task_names() -> set: diff --git a/lib/server.sh b/lib/server.sh index d1368e1..c80fdbf 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -35,14 +35,7 @@ start_server() { return fi - local gpu_args - if [[ "$runtime" == "docker-rocm" ]]; then - gpu_args=(--device=/dev/kfd --device=/dev/dri --group-add video - --cap-add SYS_PTRACE --security-opt seccomp=unconfined) - else - gpu_args=(--gpus all) - fi - local docker_args=("${gpu_args[@]}" --ipc=host --ulimit nofile=65536:65536 + local docker_args=(--gpus all --ipc=host --ulimit nofile=65536:65536 -e VLLM_ENGINE_READY_TIMEOUT_S=3600 -p "${port}:${port}") local hf_home="" From fddb9c17ce6f8a68f3cbbd4812caa04c5f6f3890 Mon Sep 17 00:00:00 2001 From: Divin Honnappa Date: Thu, 4 Jun 2026 11:26:39 -0500 Subject: [PATCH 16/62] Change agent queue to default in pipeline.yaml Signed-off-by: Divin Honnappa Signed-off-by: Stacy Roberts --- .buildkite/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index c287cae..d08d448 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,7 +1,7 @@ steps: - label: ":pipeline: generate steps" agents: - queue: small_cpu_queue_premerge + queue: default commands: - python3 -m ensurepip --upgrade --default-pip 2>/dev/null || true - python3 -m pip install --user pyyaml From abe2c31772c556d464d4e9a28caff0eaa1ddd9e1 Mon Sep 17 00:00:00 2001 From: Divin Honnappa Date: Thu, 4 Jun 2026 12:11:48 -0500 Subject: [PATCH 17/62] change queue in profile for testing Signed-off-by: Divin Honnappa Signed-off-by: Stacy Roberts --- lib/gpu_profiles.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 65de5d9..a613700 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -15,7 +15,7 @@ B200: NCCL_CUMEM_HOST_ENABLE: 0 MI355X: - queue: amd_mi355_8 + queue: default image_repo: vllm/vllm-openai-rocm server_runtime: native k8s_plugin: amd From dbd8dc7c21a572ae1eee110bd94975ab089e3b61 Mon Sep 17 00:00:00 2001 From: dhonnappa-amd Date: Fri, 5 Jun 2026 09:52:03 -0500 Subject: [PATCH 18/62] add image pull secret Signed-off-by: dhonnappa-amd Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 036e953..b431fad 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -169,6 +169,9 @@ def mi355x_k8s_plugin(image, num_gpus, profile=None): return { "kubernetes": { "podSpec": { + "imagePullSecrets": [ + {"name": "docker-config"}, + ], "containers": [ { "image": image, From e85197e32639db14c79343daac74ab87cb0e07cf Mon Sep 17 00:00:00 2001 From: dhonnappa-amd Date: Fri, 5 Jun 2026 09:55:19 -0500 Subject: [PATCH 19/62] restore queue to match upstream Signed-off-by: dhonnappa-amd Signed-off-by: Stacy Roberts --- lib/gpu_profiles.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index a613700..65de5d9 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -15,7 +15,7 @@ B200: NCCL_CUMEM_HOST_ENABLE: 0 MI355X: - queue: default + queue: amd_mi355_8 image_repo: vllm/vllm-openai-rocm server_runtime: native k8s_plugin: amd From fdebf9ce7eb457a5608ec44a93f56e34647c3685 Mon Sep 17 00:00:00 2001 From: Divin Honnappa Date: Fri, 5 Jun 2026 10:23:00 -0500 Subject: [PATCH 20/62] update secret name Signed-off-by: Divin Honnappa Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index b431fad..4e35436 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -139,7 +139,7 @@ def b200_k8s_plugin(image, num_gpus, profile=None): "name": "HF_TOKEN", "valueFrom": { "secretKeyRef": { - "name": "hf-token-secret", + "name": "hf-token", "key": "token", }, }, From 842dbcb6ebaaf6f1b2ef629f664503ac8b0c958e Mon Sep 17 00:00:00 2001 From: Divin Honnappa Date: Fri, 5 Jun 2026 10:30:51 -0500 Subject: [PATCH 21/62] fix secret name for amd Signed-off-by: Divin Honnappa Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 4e35436..2eb33d1 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -139,7 +139,7 @@ def b200_k8s_plugin(image, num_gpus, profile=None): "name": "HF_TOKEN", "valueFrom": { "secretKeyRef": { - "name": "hf-token", + "name": "hf-token-secret", "key": "token", }, }, @@ -191,7 +191,7 @@ def mi355x_k8s_plugin(image, num_gpus, profile=None): "name": "HF_TOKEN", "valueFrom": { "secretKeyRef": { - "name": "hf-token-secret", + "name": "hf-token", "key": "token", }, }, From 02ba65340d27773889700c619db05cec7d4964c9 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Fri, 5 Jun 2026 20:31:31 +0000 Subject: [PATCH 22/62] use unique queue name Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- lib/gpu_profiles.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 65de5d9..77febdb 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -15,7 +15,7 @@ B200: NCCL_CUMEM_HOST_ENABLE: 0 MI355X: - queue: amd_mi355_8 + queue: amd_mi300_perf_eval image_repo: vllm/vllm-openai-rocm server_runtime: native k8s_plugin: amd From 5c9c835c893d2a48a5a0107457efe479d46d0564 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Fri, 5 Jun 2026 21:23:22 +0000 Subject: [PATCH 23/62] cleanup Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 20 ++++++++------------ lib/parse_workload.py | 15 ++++++++------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 2eb33d1..2ccfa6d 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -20,6 +20,7 @@ import glob import os +import re import sys import yaml @@ -78,22 +79,17 @@ def is_truthy(value): def commit_from_image(image): - """Extract the commit SHA from a 'repo:nightly-' image tag.""" - ref = image.split("@", 1)[0] - tag = ref.rpartition(":")[2] - prefix = "nightly-" - if not tag.startswith(prefix): + """Extract a commit SHA from an image tag, if one is embedded.""" + _, sep, tag = image.rpartition(":") + if not sep: return "" - # tag is "nightly-"; drop any trailing "-suffix" (e.g. -cu124). - return tag[len(prefix):].split("-", 1)[0] + tag = tag.split("@", 1)[0] + m = (re.match(r"nightly-([0-9a-f]{7,40})(?:[-_.].*)?$", tag, re.IGNORECASE) + or re.search(r"(?:^|[-_.])([0-9a-f]{12,40})(?:$|[-_.])", tag, re.IGNORECASE)) + return m.group(1) if m else "" def resolved_image(data, profile): - """Resolve the docker image for the k8s pod (mirrors parse_workload). - - Honors a profile ``image_repo`` (e.g. ROCm) the same way the in-job parser - does, so the pod pulls the right repo for the hardware. - """ vllm = data.get("vllm") or {} override_image = (os.environ.get("VLLM_IMAGE") or "").strip() override_commit = (os.environ.get("VLLM_COMMIT") or "").strip() diff --git a/lib/parse_workload.py b/lib/parse_workload.py index ff119be..a6e7bbc 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -13,6 +13,7 @@ """ import os +import re import shlex import sys @@ -59,14 +60,14 @@ def env_truthy(name: str) -> bool: def commit_from_image(image: str) -> str: - """Extract the commit SHA from a 'repo:nightly-' image tag.""" - ref = image.split("@", 1)[0] # drop any "@sha256:..." digest first - tag = ref.rpartition(":")[2] # text after the tag separator - prefix = "nightly-" - if not tag.startswith(prefix): + """Extract a commit SHA from an image tag, if one is embedded.""" + _, sep, tag = image.rpartition(":") + if not sep: return "" - # tag is "nightly-"; drop any trailing "-suffix" (e.g. -cu124). - return tag[len(prefix):].split("-", 1)[0] + tag = tag.split("@", 1)[0] + m = (re.match(r"nightly-([0-9a-f]{7,40})(?:[-_.].*)?$", tag, re.IGNORECASE) + or re.search(r"(?:^|[-_.])([0-9a-f]{12,40})(?:$|[-_.])", tag, re.IGNORECASE)) + return m.group(1) if m else "" def known_task_names() -> set: From 5bcfbdeb4a111c144268e190326742e07f6143ea Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Fri, 5 Jun 2026 21:24:54 +0000 Subject: [PATCH 24/62] cleanup Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- workloads/kimi_k2_5_mi355x.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workloads/kimi_k2_5_mi355x.yaml b/workloads/kimi_k2_5_mi355x.yaml index 22fb2aa..c367f32 100644 --- a/workloads/kimi_k2_5_mi355x.yaml +++ b/workloads/kimi_k2_5_mi355x.yaml @@ -2,8 +2,7 @@ name: kimi_k2_5-mi355x gpu: MI355X num_gpus: 8 -# Opt-in for now: trigger explicitly with WORKLOADS=kimi_k2_5_mi355x until the -# MI355X path is proven, then flip to nightly: true to join the schedule. +# Opt-in for now: WORKLOADS=kimi_k2_5_mi355x nightly: false vllm: From af5d554e5d1b75f55a4c9ecebad285a131612234 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Tue, 9 Jun 2026 00:13:20 -0500 Subject: [PATCH 25/62] add mi300 * add mi300x Signed-off-by: Micah Williamson * ensure correct k8s plugin is used Signed-off-by: Micah Williamson * hardcode builder for testing Signed-off-by: Micah Williamson * revert hardcoded builder Signed-off-by: Micah Williamson --------- Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 7 ++++- .buildkite/pipeline.yaml | 2 +- lib/gpu_profiles.yaml | 10 ++++++- workloads/kimi_k2_5_mi300x.yaml | 47 +++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 workloads/kimi_k2_5_mi300x.yaml diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 2ccfa6d..ec89772 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -261,7 +261,12 @@ def make_step(path, data, profiles): "artifact_paths": ["results/**/*"], } if profile.get("server_runtime") == "native": - kind = profile.get("k8s_plugin", "nvidia") + kind = profile.get("k8s_plugin") + if not kind: + sys.exit( + f"{path}: profile {gpu!r} sets server_runtime: native but no" + f" k8s_plugin; set one explicitly (have {', '.join(K8S_PLUGINS)})" + ) builder = K8S_PLUGINS.get(kind) if builder is None: sys.exit(f"{path}: unknown k8s_plugin {kind!r} (have {', '.join(K8S_PLUGINS)})") diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index d08d448..c287cae 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,7 +1,7 @@ steps: - label: ":pipeline: generate steps" agents: - queue: default + queue: small_cpu_queue_premerge commands: - python3 -m ensurepip --upgrade --default-pip 2>/dev/null || true - python3 -m pip install --user pyyaml diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 77febdb..63429b4 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -10,12 +10,20 @@ B200: queue: b200-k8s hf_home: /mnt/shared/hf_cache server_runtime: native + k8s_plugin: nvidia env: VLLM_DEEP_GEMM_WARMUP: skip NCCL_CUMEM_HOST_ENABLE: 0 +MI300X: + queue: mi300_perf_eval + image_repo: vllm/vllm-openai-rocm + server_runtime: native + k8s_plugin: amd + hf_home: /mnt/shared/hf-models + MI355X: - queue: amd_mi300_perf_eval + queue: mi355_perf_eval image_repo: vllm/vllm-openai-rocm server_runtime: native k8s_plugin: amd diff --git a/workloads/kimi_k2_5_mi300x.yaml b/workloads/kimi_k2_5_mi300x.yaml new file mode 100644 index 0000000..8c4d1ab --- /dev/null +++ b/workloads/kimi_k2_5_mi300x.yaml @@ -0,0 +1,47 @@ +# Kimi-K2.5 on AMD MI300X +name: kimi_k2_5-mi300x +gpu: MI300X +num_gpus: 8 +# Opt-in for now: WORKLOADS=kimi_k2_5_mi300x +nightly: false + +vllm: + model: moonshotai/Kimi-K2.5 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + num_threads: 8 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From 53029ff96f9cb073a204b1ce465add301828450e Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Tue, 9 Jun 2026 05:29:30 +0000 Subject: [PATCH 26/62] add DS V4 config for mi355 Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- .buildkite/generate_pipeline.py | 5 ++-- workloads/deepseek_v4_pro_mi355x.yaml | 41 +++++++++++++++++++++++++++ workloads/kimi_k2_5_mi300x.yaml | 8 ------ workloads/kimi_k2_5_mi355x.yaml | 12 -------- 4 files changed, 44 insertions(+), 22 deletions(-) create mode 100644 workloads/deepseek_v4_pro_mi355x.yaml diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index ec89772..60adcb1 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -59,6 +59,7 @@ def setup_command(packages): "B200": ":b200:", "A100": ":a100:", "MI355X": ":amd:", + "MI300X": ":amd:", } ECR_PUBLIC_PREFIX = "public.ecr.aws/" @@ -159,7 +160,7 @@ def b200_k8s_plugin(image, num_gpus, profile=None): } -def mi355x_k8s_plugin(image, num_gpus, profile=None): +def amd_k8s_plugin(image, num_gpus, profile=None): profile = profile or {} hf_home = profile.get("hf_home") or "/root/.cache/huggingface" return { @@ -209,7 +210,7 @@ def mi355x_k8s_plugin(image, num_gpus, profile=None): K8S_PLUGINS = { "nvidia": b200_k8s_plugin, - "amd": mi355x_k8s_plugin, + "amd": amd_k8s_plugin, } diff --git a/workloads/deepseek_v4_pro_mi355x.yaml b/workloads/deepseek_v4_pro_mi355x.yaml new file mode 100644 index 0000000..9a52e57 --- /dev/null +++ b/workloads/deepseek_v4_pro_mi355x.yaml @@ -0,0 +1,41 @@ +# DeepSeek-V4-Pro on MI355X +name: deepseek_v4_pro-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=deepseek_v4_pro_mi355x +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-V4-Pro + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --kv-cache-dtype fp8 + --trust-remote-code + --no-enable-prefix-caching + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/kimi_k2_5_mi300x.yaml b/workloads/kimi_k2_5_mi300x.yaml index 8c4d1ab..bb30dc9 100644 --- a/workloads/kimi_k2_5_mi300x.yaml +++ b/workloads/kimi_k2_5_mi300x.yaml @@ -26,14 +26,6 @@ lm_eval: max_length: 40960 max_gen_toks: 32768 -bfcl: - test_categories: - - simple_python - - multiple - - parallel - - parallel_multiple - num_threads: 8 - vllm_bench: configs: - name: 8k-in-1k-out-conc-128 diff --git a/workloads/kimi_k2_5_mi355x.yaml b/workloads/kimi_k2_5_mi355x.yaml index c367f32..e5ec183 100644 --- a/workloads/kimi_k2_5_mi355x.yaml +++ b/workloads/kimi_k2_5_mi355x.yaml @@ -11,10 +11,6 @@ vllm: VLLM_ROCM_USE_AITER: 1 serve_args: >- --tensor-parallel-size 8 - --mm-encoder-tp-mode data - --tool-call-parser kimi_k2 - --reasoning-parser kimi_k2 - --enable-auto-tool-choice --trust-remote-code lm_eval: @@ -30,14 +26,6 @@ lm_eval: max_length: 40960 max_gen_toks: 32768 -bfcl: - test_categories: - - simple_python - - multiple - - parallel - - parallel_multiple - num_threads: 8 - vllm_bench: configs: - name: 8k-in-1k-out-conc-128 From b335d915722690614cdd20b93297131e1447d319 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Tue, 9 Jun 2026 06:10:27 +0000 Subject: [PATCH 27/62] add quantization flag for ds v4 Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- workloads/deepseek_v4_pro_mi355x.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workloads/deepseek_v4_pro_mi355x.yaml b/workloads/deepseek_v4_pro_mi355x.yaml index 9a52e57..3e74a9e 100644 --- a/workloads/deepseek_v4_pro_mi355x.yaml +++ b/workloads/deepseek_v4_pro_mi355x.yaml @@ -14,6 +14,7 @@ vllm: --kv-cache-dtype fp8 --trust-remote-code --no-enable-prefix-caching + --quantization deepseek_v4_fp8 lm_eval: model_args: From 45a168db6c1ee03cdb184008c26bf4fc889a3463 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Tue, 9 Jun 2026 06:41:42 +0000 Subject: [PATCH 28/62] add gpt-oss config for mi355 Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- workloads/gpt_oss_120b_mi355x.yaml | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 workloads/gpt_oss_120b_mi355x.yaml diff --git a/workloads/gpt_oss_120b_mi355x.yaml b/workloads/gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..dd99890 --- /dev/null +++ b/workloads/gpt_oss_120b_mi355x.yaml @@ -0,0 +1,38 @@ +# GPT-OSS 120B on MI355X +name: gpt_oss_120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From 4800994dbfdbe20216096c408fb6807d025db7ab Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Tue, 9 Jun 2026 06:44:55 +0000 Subject: [PATCH 29/62] enable aiter for gpt oss Signed-off-by: Micah Williamson Signed-off-by: Stacy Roberts --- workloads/gpt_oss_120b_mi355x.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workloads/gpt_oss_120b_mi355x.yaml b/workloads/gpt_oss_120b_mi355x.yaml index dd99890..3b01ef2 100644 --- a/workloads/gpt_oss_120b_mi355x.yaml +++ b/workloads/gpt_oss_120b_mi355x.yaml @@ -7,6 +7,8 @@ nightly: false vllm: model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 serve_args: >- --tensor-parallel-size 1 --trust-remote-code From a4c14d33f9240f9565c7393a85d2efdbc9db0f79 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Thu, 11 Jun 2026 12:52:52 +0000 Subject: [PATCH 30/62] Updated to allow for testing various backend attentions with models. Optional addition, not requiring any updates to current yaml files. Signed-off-by: Stacy Roberts --- README.md | 6 +- lib/parse_workload.py | 50 ++++-- lib/run.sh | 145 +++++++++++------- lib/server.sh | 3 + workloads/gpt_oss_120b_mi355x_attn_sweep.yaml | 42 +++++ 5 files changed, 172 insertions(+), 74 deletions(-) create mode 100644 workloads/gpt_oss_120b_mi355x_attn_sweep.yaml diff --git a/README.md b/README.md index 93bfb1b..6e729fb 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ CLAUDE.md agent conventions and detailed Buildkite workflow A recipe has top-level metadata plus up to three eval blocks: -- **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides). Required. +- **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides, optional `attention_backends` list). Required. - **`lm_eval:`** — *what accuracy to measure.* Lists lm-evaluation-harness tasks to run against the live server (e.g. `gsm8k`, `aime25`). Each task's score is saved under `results///`. Optional. - **`vllm_bench:`** — *what perf to measure.* Lists `vllm bench serve` configs (input/output lengths, concurrency, dataset). Raw JSON is saved and ingested into the perf dashboard. Optional. - **`bfcl:`** — *function-calling eval.* Runs [BFCL](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) test categories against the live server. Some models need `--enable-auto-tool-choice` and `--tool-call-parser` in `serve_args`. Results are transformed to lm_eval format and ingested as `bfcl_` tasks. Optional. @@ -47,6 +47,9 @@ vllm: # how the server is brought up serve_args: >- # appended to `vllm serve `; word-split -dp 8 --enable-expert-parallel --trust-remote-code + attention_backends: # optional; list of VLLM_ATTENTION_BACKEND values + - FLASH_ATTN # when set, the full eval suite runs once per + - FLASHINFER # backend; results land in attn-/ subdirs lm_eval: # accuracy tasks (optional) model_args: # workload-level defaults, merged into every task @@ -81,6 +84,7 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: +- **`vllm.attention_backends`** is an optional list of vLLM attention backend names (`FLASH_ATTN`, `FLASHINFER`, `XFORMERS`, `TRITON_ATTN`, `TRITON_MLA`, `ROCM_FLASH`, `PAGED_ATTENTION`,`ROCM_AITER_FA`,`ROCM_AITER_UNIFIED_ATTN`, `ROCM_ATTN`,`ROCM_AITER_MLA`,`ROCM_AITER_MLA_SPARSE`, `ROCM_AITER_TRITON_MLA`). When set, the orchestrator starts the server once per backend — adding `--attention-backend $ATTN_BACKEND` — and runs the complete eval suite (bench, lm_eval, bfcl) for each. Results are stored under `results//attn-/` so every backend gets its own isolated output directory. Without this field, the server starts once with whatever attention backend vLLM selects by default and results go to `results//` as usual. See `workloads/gpt_oss_120b_mi355x_attn_sweep.yaml` for an example. - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. diff --git a/lib/parse_workload.py b/lib/parse_workload.py index a6e7bbc..23a5878 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -20,6 +20,15 @@ import yaml TASK_FIELDS = {"name", "num_fewshot", "model_args"} +VLLM_FIELDS = { + "model", "image", "serve_args", "env", "attention_backends", +} +KNOWN_ATTENTION_BACKENDS = { + "FLASH_ATTN", "FLASHINFER", "XFORMERS", "TRITON_ATTN", + "TRITON_MLA", "ROCM_FLASH", "PAGED_ATTENTION", "ROCM_AITER_FA", + "ROCM_AITER_UNIFIED_ATTN", "ROCM_ATTN", "ROCM_AITER_MLA", + "ROCM_AITER_MLA_SPARSE", "ROCM_AITER_TRITON_MLA" +} BENCH_FIELDS = { "name", "backend", "dataset", "input_len", "output_len", "num_prompts", "max_concurrency", @@ -38,6 +47,7 @@ "all", "all_scoring", "single_turn", "multi_turn", "live", "non_live", "non_python", "python", "memory", "agentic", } +COMMIT_IMAGE_TEMPLATE = "vllm/vllm-openai:nightly-{commit}" def emit(name: str, value: object) -> None: @@ -88,23 +98,15 @@ def load_profile(gpu: str, workload_path: str) -> dict: return profiles[gpu] -def resolve_image(vllm: dict, profile: dict) -> tuple[str, str]: +def resolve_image(vllm: dict) -> tuple[str, str]: """Pick the image and commit using VLLM_IMAGE / VLLM_COMMIT / workload.""" override_image = (os.environ.get("VLLM_IMAGE") or "").strip() override_commit = (os.environ.get("VLLM_COMMIT") or "").strip() - # ROCm images are located at vllm/vllm-openai-rocm. The default - # images (CUDA) are stored at vllm/vllm-openai - custom_repo = (profile.get("image_repo") or "").strip() - repo = custom_repo or "vllm/vllm-openai" - - if override_image and (not custom_repo or repo in override_image): + if override_image: return override_image, override_commit or commit_from_image(override_image) - - commit = override_commit or commit_from_image(override_image) - if commit: - return f"{repo}:nightly-{commit}", commit - - image = vllm.get("image", f"{repo}:nightly") + if override_commit: + return COMMIT_IMAGE_TEMPLATE.format(commit=override_commit), override_commit + image = vllm.get("image", "vllm/vllm-openai:nightly") return image, commit_from_image(str(image)) @@ -222,6 +224,19 @@ def bfcl_tsv(bfcl: dict) -> str: return "\n".join(f"{cat}\t{num_threads}\t{temperature}" for cat in cats) +def validate_attention_backends(backends: list, path: str) -> None: + if not backends: + sys.exit(f"{path}: vllm.attention_backends must not be empty if specified") + for b in backends: + if b not in KNOWN_ATTENTION_BACKENDS: + sys.exit( + f"{path}: unknown attention backend {b!r}; " + f"known: {', '.join(sorted(KNOWN_ATTENTION_BACKENDS))}" + ) + if len(backends) != len(set(backends)): + sys.exit(f"{path}: duplicate entries in vllm.attention_backends") + + def main(path: str) -> None: with open(path) as f: data = yaml.safe_load(f) @@ -248,7 +263,11 @@ def main(path: str) -> None: if bfcl: validate_bfcl(bfcl, serve_args, path) - image, vllm_commit = resolve_image(vllm, profile) + attention_backends = vllm.get("attention_backends") or [] + if attention_backends: + validate_attention_backends(attention_backends, path) + + image, vllm_commit = resolve_image(vllm) env = {**(profile.get("env") or {}), **(vllm.get("env") or {})} if "HF_HOME" not in env and profile.get("hf_home"): env["HF_HOME"] = profile["hf_home"] @@ -271,6 +290,9 @@ def main(path: str) -> None: emit("BENCH_DEVICE", metadata.get("device") or gpu.lower()) emit("BENCH_TP", tp) emit("BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or "")) + # One backend per line; empty string when not specified (run.sh treats this + # as a single "default" pass with no VLLM_ATTENTION_BACKEND override). + emit("ATTENTION_BACKENDS", "\n".join(attention_backends)) if __name__ == "__main__": diff --git a/lib/run.sh b/lib/run.sh index 96ce957..92e5b4c 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -2,6 +2,11 @@ # Orchestrate a workload: bring up vLLM, then dispatch each task to the # helper script for its type. # +# When vllm.attention_backends is set in the workload YAML, the server is +# started once per backend (with --attention-backend appended to serve_args), +# and the full eval suite runs for each. Results land in +# results//attn-/ instead of results//. +# # Usage: ./lib/run.sh workloads/qwen3_5_h200.yaml set -euo pipefail @@ -20,70 +25,92 @@ eval "$WORKLOAD_EXPORTS" export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME PORT=8000 -CONTAINER="perf-eval-${WORKLOAD_NAME}-$$" -RESULTS_DIR="results/${WORKLOAD_NAME}" BASE_URL="http://localhost:${PORT}" BENCH_TRUST_REMOTE_CODE=false if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then BENCH_TRUST_REMOTE_CODE=true fi -mkdir -p "$RESULTS_DIR" - -trap 'stop_server "$CONTAINER"' EXIT - -start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ - "$WORKLOAD_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" -wait_healthy "$PORT" - -# vllm bench serve runs first so we can validate perf flow without waiting -# on a full lm_eval pass. Each config's raw json lands in -# $RESULTS_DIR/bench-.json and is then transformed and POSTed to the -# perf dashboard ingest endpoint. -while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do - [[ -z "$bname" ]] && continue - run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ - "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ - "$conc" "$speed_subset" "$speed_category" \ - "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR" - - python3 "$DIR/ingest_perf.py" \ - --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ - --device "$WORKLOAD_BENCH_DEVICE" \ - --tp "$WORKLOAD_BENCH_TP" \ - --precision "$WORKLOAD_BENCH_PRECISION" \ - --model "$WORKLOAD_MODEL" \ - --image "$WORKLOAD_IMAGE" \ - --isl "$isl" --osl "$osl" --conc "$conc" || true -done <<< "$WORKLOAD_VLLM_BENCH_TSV" - -if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then - echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" - exit 0 + +# Build the list of attention backends to sweep. An empty +# WORKLOAD_ATTENTION_BACKENDS means "run once with whatever vLLM picks" and +# we represent that as a single sentinel entry so the loop always executes. +mapfile -t ATTN_BACKENDS <<< "${WORKLOAD_ATTENTION_BACKENDS}" +if [[ "${#ATTN_BACKENDS[@]}" -eq 0 || ( "${#ATTN_BACKENDS[@]}" -eq 1 && -z "${ATTN_BACKENDS[0]}" ) ]]; then + ATTN_BACKENDS=("default") fi -while IFS=$'\t' read -r task fewshot model_args; do - [[ -z "$task" ]] && continue - run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ - "$model_args" "$RESULTS_DIR" - - python3 "$DIR/ingest.py" \ - --results-dir "${RESULTS_DIR}/${task}" \ - --workload "$WORKLOAD_NAME" \ - --task "$task" \ - ${INGEST_NO_SAMPLES:+--no-samples} || true -done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" - -# bfcl function-calling eval -while IFS=$'\t' read -r category num_threads temperature; do - [[ -z "$category" ]] && continue - echo "--- :phone: bfcl ${category}" - python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ - "$category" "$num_threads" "$temperature" "$RESULTS_DIR" - - python3 "$DIR/ingest.py" \ - --results-dir "${RESULTS_DIR}/bfcl-${category}" \ - --workload "$WORKLOAD_NAME" \ - --task "bfcl_${category}" \ - --no-samples || true -done <<< "$WORKLOAD_BFCL_TSV" +for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do + if [[ "$ATTN_BACKEND" == "default" ]]; then + echo "=== :brain: attention backend: (vLLM default)" + RESULTS_DIR="results/${WORKLOAD_NAME}" + EFFECTIVE_SERVE_ARGS="$WORKLOAD_SERVE_ARGS" + else + echo "=== :brain: attention backend: ${ATTN_BACKEND}" + RESULTS_DIR="results/${WORKLOAD_NAME}/attn-${ATTN_BACKEND}" + # --attention-backend is a vLLM server arg, not an env var. + EFFECTIVE_SERVE_ARGS="${WORKLOAD_SERVE_ARGS} --attention-backend ${ATTN_BACKEND}" + fi + mkdir -p "$RESULTS_DIR" + + CONTAINER="perf-eval-${WORKLOAD_NAME}-${ATTN_BACKEND}-$$" + + trap 'stop_server "$CONTAINER"' EXIT + + start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" + wait_healthy "$PORT" + + # vllm bench serve runs first so we can validate perf flow without waiting + # on a full lm_eval pass. Each config's raw json lands in + # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the + # perf dashboard ingest endpoint. + while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do + [[ -z "$bname" ]] && continue + run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR" + + python3 "$DIR/ingest_perf.py" \ + --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ + --device "$WORKLOAD_BENCH_DEVICE" \ + --tp "$WORKLOAD_BENCH_TP" \ + --precision "$WORKLOAD_BENCH_PRECISION" \ + --model "$WORKLOAD_MODEL" \ + --image "$WORKLOAD_IMAGE" \ + --isl "$isl" --osl "$osl" --conc "$conc" || true + done <<< "$WORKLOAD_VLLM_BENCH_TSV" + + if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" + exit 0 + fi + + while IFS=$'\t' read -r task fewshot model_args; do + [[ -z "$task" ]] && continue + run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ + "$model_args" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/${task}" \ + --workload "$WORKLOAD_NAME" \ + --task "$task" \ + ${INGEST_NO_SAMPLES:+--no-samples} || true + done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + + # bfcl function-calling eval + while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true + done <<< "$WORKLOAD_BFCL_TSV" + +done << "$WORKLOAD_BFCL_TSV" diff --git a/lib/server.sh b/lib/server.sh index c80fdbf..77eb29e 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -10,6 +10,9 @@ # is also bind-mounted at the same path inside the container so the model cache # on the host is visible to vLLM. For native runtime, values are exported before # starting `vllm serve` in the current job container. +# Note: attention backend selection is passed via --attention-backend in +# serve_args, not as an environment variable (vLLM does not support +# VLLM_ATTENTION_BACKEND as an env var). # # After start_server, vLLM logs are streamed to stdout (prefixed with `[vllm]`) # so build output reflects server startup progress in real time. The streamer's diff --git a/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml new file mode 100644 index 0000000..eb3c499 --- /dev/null +++ b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml @@ -0,0 +1,42 @@ +# GPT-OSS 120B on MI355X +name: attn_sweep_gpt_oss_120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + attention_backends: + - ROCM_AITER_UNIFIED_ATTN + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 1024 + output_len: 1024 + num_prompts: 5 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy From 16a2328e8de33ec79ea0743de2aa9c9fc87fd9d9 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Thu, 11 Jun 2026 19:43:30 +0000 Subject: [PATCH 31/62] added delay between server stop and server start for gpu spin down between attention set ups. (will also happen at end if single attention run) Signed-off-by: Stacy Roberts --- .gitattributes | 3 +++ lib/run.sh | 13 +++++++++++-- lib/server.sh | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3621a72 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf diff --git a/lib/run.sh b/lib/run.sh index 92e5b4c..901eca3 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -60,6 +60,9 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" wait_healthy "$PORT" + #if [[ "$ATTN_BACKEND" != "default" ]]; then + # verify_attention_backend "$CONTAINER" "$ATTN_BACKEND" "$WORKLOAD_SERVER_RUNTIME" + #fi # vllm bench serve runs first so we can validate perf flow without waiting # on a full lm_eval pass. Each config's raw json lands in @@ -84,7 +87,10 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" - exit 0 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue fi while IFS=$'\t' read -r task fewshot model_args; do @@ -113,4 +119,7 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do --no-samples || true done <<< "$WORKLOAD_BFCL_TSV" -done << "$WORKLOAD_BFCL_TSV" + stop_server "$CONTAINER" + trap - EXIT + drain_gpu +done diff --git a/lib/server.sh b/lib/server.sh index 77eb29e..cbda134 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -30,6 +30,7 @@ start_server() { local log_file="/tmp/${container}.log" VLLM_LOG_FILE="$log_file" # shellcheck disable=SC2086 # serve_args intentionally word-split + echo "Server call: vllm server $model --port $port $serve_args" vllm serve "$model" --port "$port" $serve_args >"$log_file" 2>&1 & VLLM_SERVER_PID=$! echo "--- :memo: streaming vllm logs" @@ -104,3 +105,36 @@ stop_server() { fi docker rm -f "$container" >/dev/null 2>&1 || true } + +# Wait until all GPUs have freed their VRAM below a low watermark, or until +# timeout. Call this between backends so the next vllm serve doesn't OOM on +# memory still held by the dying container's ROCm context. +# rocm-smi --showmeminfo vram emits lines like: +# GPU[0] : VRAM Total Used Memory (B): 12345678 +drain_gpu() { + local timeout=${1:-120} threshold_gib=${2:-1} + local threshold_bytes=$(( threshold_gib * 1024 * 1024 * 1024 )) + echo "--- :hourglass: waiting for GPU VRAM to drain (threshold ${threshold_gib} GiB, timeout ${timeout}s)" + local deadline + deadline=$(( $(date +%s) + timeout )) + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "rocm-smi unavailable; skipping GPU drain check" >&2 + return 0 + fi + while (( $(date +%s) < deadline )); do + local max_used + max_used=$(rocm-smi --showmeminfo vram --noheader 2>/dev/null \ + | awk '/VRAM Total Used Memory/{if($NF+0>m)m=$NF+0} END{print m+0}') || true + if [[ -z "$max_used" ]]; then + echo "rocm-smi returned no data; skipping GPU drain check" >&2 + return 0 + fi + if (( max_used < threshold_bytes )); then + echo "GPU VRAM drained (max used: $(( max_used / 1024 / 1024 )) MiB)" + return 0 + fi + echo "GPU still holds $(( max_used / 1024 / 1024 )) MiB VRAM; waiting..." + sleep 5 + done + echo "WARNING: GPU VRAM did not drain within ${timeout}s (max used: $(( max_used / 1024 / 1024 )) MiB); proceeding anyway" >&2 +} \ No newline at end of file From d78ee119cd6af72e04cd9e89ade9912bc64bcd6b Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Thu, 11 Jun 2026 19:59:19 +0000 Subject: [PATCH 32/62] clean up Signed-off-by: Stacy Roberts --- lib/run.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/run.sh b/lib/run.sh index 901eca3..d009ddb 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -6,8 +6,7 @@ # started once per backend (with --attention-backend appended to serve_args), # and the full eval suite runs for each. Results land in # results//attn-/ instead of results//. -# -# Usage: ./lib/run.sh workloads/qwen3_5_h200.yaml + set -euo pipefail WORKLOAD="${1:?usage: $0 }" @@ -60,9 +59,6 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" wait_healthy "$PORT" - #if [[ "$ATTN_BACKEND" != "default" ]]; then - # verify_attention_backend "$CONTAINER" "$ATTN_BACKEND" "$WORKLOAD_SERVER_RUNTIME" - #fi # vllm bench serve runs first so we can validate perf flow without waiting # on a full lm_eval pass. Each config's raw json lands in From cda22c567c27ac3da0ab57bed4b4fc8f96118234 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Fri, 12 Jun 2026 21:57:12 +0000 Subject: [PATCH 33/62] yamls for a few AFO-LLM workloads, server.sh trying to get compilation-config working (it doesn't yet) Signed-off-by: Stacy Roberts --- lib/server.sh | 13 +- workloads/gpt_oss_120b_mi355x_attn_sweep.yaml | 1 + .../upstream_deepseek_r1_0528_mi355x.yaml | 102 +++++++++++++++ .../upstream_deepseek_r1_mxfp4_mi355x.yaml | 91 ++++++++++++++ workloads/upstream_gpt_oss_20b_mi355x.yaml | 117 ++++++++++++++++++ 5 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 workloads/upstream_deepseek_r1_0528_mi355x.yaml create mode 100644 workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml create mode 100644 workloads/upstream_gpt_oss_20b_mi355x.yaml diff --git a/lib/server.sh b/lib/server.sh index cbda134..53d68ab 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -30,8 +30,12 @@ start_server() { local log_file="/tmp/${container}.log" VLLM_LOG_FILE="$log_file" # shellcheck disable=SC2086 # serve_args intentionally word-split - echo "Server call: vllm server $model --port $port $serve_args" - vllm serve "$model" --port "$port" $serve_args >"$log_file" 2>&1 & + #echo "Server call: vllm server $model --port $port $serve_args" + #vllm serve "$model" --port "$port" $serve_args >"$log_file" 2>&1 & + local -a serve_args_arr + IFS=' ' read -ra serve_args_arr <<< "$serve_args" + echo "Server call: vllm serve $model --port $port ${serve_args_arr[*]}" + vllm serve "$model" --port "$port" "${serve_args_arr[@]}" >"$log_file" 2>&1 & VLLM_SERVER_PID=$! echo "--- :memo: streaming vllm logs" ( tail -f "$log_file" 2>/dev/null | stdbuf -oL -eL sed 's/^/[vllm] /' ) & @@ -53,11 +57,14 @@ start_server() { fi # shellcheck disable=SC2086 # serve_args intentionally word-split + local -a server_args_arr + IFS=' ' read -ra serve_args_arr <<< "$serve_args" # vllm/vllm-openai's entrypoint takes the model as the first positional # arg; do not prepend `vllm` or `serve`. docker run -d --rm --name "$container" "${docker_args[@]}" \ "$image" \ - "$model" --port "$port" $serve_args + # "$model" --port "$port" $serve_args + "$model" --port "$port" "${serve_args_arr[@]}" echo "--- :memo: streaming vllm logs" ( docker logs -f "$container" 2>&1 | stdbuf -oL -eL sed 's/^/[vllm] /' ) & diff --git a/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml index eb3c499..9cf4df1 100644 --- a/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml +++ b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml @@ -15,6 +15,7 @@ vllm: --kv-cache-dtype=fp8 attention_backends: - ROCM_AITER_UNIFIED_ATTN + - ROCM_AITER_FA lm_eval: model_args: diff --git a/workloads/upstream_deepseek_r1_0528_mi355x.yaml b/workloads/upstream_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..57c831f --- /dev/null +++ b/workloads/upstream_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,102 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: upstream-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + env: + VLLM_USE_V1: 1 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml b/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..f7fc3f5 --- /dev/null +++ b/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,91 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# max_num_seqs=32 (much lower than other groups — intentional per original config). +# extra_args: async-scheduling, kv-cache-dtype=fp8, block-size=1, compilation-config +# The compilation-config JSON is passed as a single quoted string in serve_args. +# No ACCURACY group for this model in vllm_upstream.yaml. +name: deepseek-r1-mxfp4-upstream-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling +# --compilation-config {"pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/upstream_gpt_oss_20b_mi355x.yaml b/workloads/upstream_gpt_oss_20b_mi355x.yaml new file mode 100644 index 0000000..4150584 --- /dev/null +++ b/workloads/upstream_gpt_oss_20b_mi355x.yaml @@ -0,0 +1,117 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# ACCURACY (TP=1) + SERVING (TP=1, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64,128]) +# extra_args block-size=64 and async-scheduling go into serve_args. +# ACCURACY uses lm_eval (gsm8k 5-shot, matching other upstream workloads). +name: upstream-gpt-oss-20b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-20b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + env: +# VLLM_USE_V1: 1 + VLLM_ROCM_USE_AITER: 1 +# VLLM_USE_AITER_UNIFIED_ATTENTION: 1 +# VLLM_ROCM_USE_AITER_MHA: 0 +# HSA_NO_SCRATCH_RECLAIM: 1 +# NCCL_MIN_NCHANNELS: 112 +# USE_FASTSAFETENSOR: 1 +# SAFETENSORS_FAST_GPU: 1 +attention_backends: + - ROCM_AITER_UNIFIED_ATTN + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 From 1e4092e148e4fcdbe366485462e00f69ef51995a Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Mon, 15 Jun 2026 18:36:08 +0000 Subject: [PATCH 34/62] env/compilation-config updated Signed-off-by: Stacy Roberts --- workloads/upstream_deepseek_r1_0528_mi355x.yaml | 7 ++++--- workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml | 4 ++-- workloads/upstream_gpt_oss_20b_mi355x.yaml | 12 ++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/workloads/upstream_deepseek_r1_0528_mi355x.yaml b/workloads/upstream_deepseek_r1_0528_mi355x.yaml index 57c831f..d8fd942 100644 --- a/workloads/upstream_deepseek_r1_0528_mi355x.yaml +++ b/workloads/upstream_deepseek_r1_0528_mi355x.yaml @@ -5,13 +5,14 @@ # same server instance handles both eval types. name: upstream-deepseek-r1-0528-MI355X gpu: MI355X -num_gpus: 8 +num_gpus: 1 #8 nightly: false +#tp should be 8 vllm: model: deepseek-ai/DeepSeek-R1-0528 serve_args: >- - --tensor-parallel-size 8 + --tensor-parallel-size 1 --dtype auto --max-num-batched-tokens 131072 --max-num-seqs 1024 @@ -20,7 +21,7 @@ vllm: --block-size 1 --async-scheduling env: - VLLM_USE_V1: 1 +# VLLM_USE_V1: 1 VLLM_ROCM_USE_AITER: 1 HSA_NO_SCRATCH_RECLAIM: 1 NCCL_MIN_NCHANNELS: 112 diff --git a/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml b/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml index f7fc3f5..c7af18d 100644 --- a/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml +++ b/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml @@ -24,8 +24,8 @@ vllm: --max-model-len 10240 --block-size 1 --async-scheduling -# --compilation-config {"pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} - + --compilation-config {"pass_config":{"eliminate_noops":false,},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} +# adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, vllm_bench: configs: diff --git a/workloads/upstream_gpt_oss_20b_mi355x.yaml b/workloads/upstream_gpt_oss_20b_mi355x.yaml index 4150584..2578038 100644 --- a/workloads/upstream_gpt_oss_20b_mi355x.yaml +++ b/workloads/upstream_gpt_oss_20b_mi355x.yaml @@ -21,12 +21,12 @@ vllm: env: # VLLM_USE_V1: 1 VLLM_ROCM_USE_AITER: 1 -# VLLM_USE_AITER_UNIFIED_ATTENTION: 1 -# VLLM_ROCM_USE_AITER_MHA: 0 -# HSA_NO_SCRATCH_RECLAIM: 1 -# NCCL_MIN_NCHANNELS: 112 -# USE_FASTSAFETENSOR: 1 -# SAFETENSORS_FAST_GPU: 1 + VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 attention_backends: - ROCM_AITER_UNIFIED_ATTN From 006d82923a6473c9c1337d7fd0aacea4b48e45cd Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Mon, 15 Jun 2026 22:21:25 +0000 Subject: [PATCH 35/62] fixed datetime issue to remove warning Signed-off-by: Stacy Roberts --- .gitignore | 165 +++++++++++++++++++++ lib/ingest_perf.py | 2 +- workloads/upstream_gpt_oss_20b_mi355x.yaml | 1 - 3 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80311f4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,165 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# results directory +results/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/lib/ingest_perf.py b/lib/ingest_perf.py index 5291d43..620a7bf 100644 --- a/lib/ingest_perf.py +++ b/lib/ingest_perf.py @@ -49,7 +49,7 @@ def transform(raw: dict, args: argparse.Namespace) -> dict: input_throughput = total_token_throughput - output_throughput data = { - "date": args.date or datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + "date": args.date or datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"), "device": args.device, "conc": int(raw.get("max_concurrency") or args.conc), "image": args.image, diff --git a/workloads/upstream_gpt_oss_20b_mi355x.yaml b/workloads/upstream_gpt_oss_20b_mi355x.yaml index 2578038..63209b5 100644 --- a/workloads/upstream_gpt_oss_20b_mi355x.yaml +++ b/workloads/upstream_gpt_oss_20b_mi355x.yaml @@ -19,7 +19,6 @@ vllm: --block-size 64 --async-scheduling env: -# VLLM_USE_V1: 1 VLLM_ROCM_USE_AITER: 1 VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 VLLM_ROCM_USE_AITER_MHA: 0 From b6386e7c51c815ebc8626ad88d1ebbc17e3d2460 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Tue, 16 Jun 2026 21:31:48 +0000 Subject: [PATCH 36/62] Updates for drain gpu utilization and multiple backends including using the default Signed-off-by: Stacy Roberts --- lib/parse_workload.py | 1 + lib/run.sh | 19 +++++++++++++++ lib/server.sh | 56 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/lib/parse_workload.py b/lib/parse_workload.py index 23a5878..e2eb900 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -24,6 +24,7 @@ "model", "image", "serve_args", "env", "attention_backends", } KNOWN_ATTENTION_BACKENDS = { + "default", "FLASH_ATTN", "FLASHINFER", "XFORMERS", "TRITON_ATTN", "TRITON_MLA", "ROCM_FLASH", "PAGED_ATTENTION", "ROCM_AITER_FA", "ROCM_AITER_UNIFIED_ATTN", "ROCM_ATTN", "ROCM_AITER_MLA", diff --git a/lib/run.sh b/lib/run.sh index d009ddb..d16e412 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -60,6 +60,22 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" wait_healthy "$PORT" +#figuring out what the default attention backend is + if [[ "$ATTN_BACKEND" == "default" ]]; then + echo "--- :mag: attention backend selected by vLLM:" + if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then + grep -E "(Using|Overriding with|Incompatible).*(backend|Backend)" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null \ + | grep "Worker_TP0\|Worker pid" \ + | sed 's/^/ /' || echo " (backend selection lines not found in log)" + else + docker logs "$CONTAINER" 2>&1 \ + | grep -E "(Using|Overriding with|Incompatible).*(backend|Backend)" \ + | grep "Worker_TP0\|Worker pid" \ + | sed 's/^/ /' || echo " (backend selection lines not found in log)" + fi + fi + # vllm bench serve runs first so we can validate perf flow without waiting # on a full lm_eval pass. Each config's raw json lands in # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the @@ -79,6 +95,9 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do --model "$WORKLOAD_MODEL" \ --image "$WORKLOAD_IMAGE" \ --isl "$isl" --osl "$osl" --conc "$conc" || true + + echo "Giving GPU time to settle between tests" + wait_gpu_idle done <<< "$WORKLOAD_VLLM_BENCH_TSV" if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then diff --git a/lib/server.sh b/lib/server.sh index 53d68ab..ed93847 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -107,6 +107,8 @@ stop_server() { wait "$VLLM_LOGS_PID" 2>/dev/null || true fi if [[ -n "${VLLM_SERVER_PID:-}" ]]; then + kill -SIGINT "$VLLM_SERVER_PID" 2>/dev/null || true + wait "$VLLM_SERVER_PID" 2>/dev/null || true kill "$VLLM_SERVER_PID" 2>/dev/null || true wait "$VLLM_SERVER_PID" 2>/dev/null || true fi @@ -118,9 +120,11 @@ stop_server() { # memory still held by the dying container's ROCm context. # rocm-smi --showmeminfo vram emits lines like: # GPU[0] : VRAM Total Used Memory (B): 12345678 +# timeout default: 120sec, threshold_gib default: 0 drain_gpu() { - local timeout=${1:-120} threshold_gib=${2:-1} + local timeout=${1:-120} threshold_gib=${2:-0} local threshold_bytes=$(( threshold_gib * 1024 * 1024 * 1024 )) + local current_used=0 echo "--- :hourglass: waiting for GPU VRAM to drain (threshold ${threshold_gib} GiB, timeout ${timeout}s)" local deadline deadline=$(( $(date +%s) + timeout )) @@ -129,19 +133,55 @@ drain_gpu() { return 0 fi while (( $(date +%s) < deadline )); do - local max_used - max_used=$(rocm-smi --showmeminfo vram --noheader 2>/dev/null \ + current_used=$(rocm-smi --showmeminfo vram --noheader 2>/dev/null \ | awk '/VRAM Total Used Memory/{if($NF+0>m)m=$NF+0} END{print m+0}') || true - if [[ -z "$max_used" ]]; then + if [[ -z "$current_used" ]]; then echo "rocm-smi returned no data; skipping GPU drain check" >&2 return 0 fi - if (( max_used < threshold_bytes )); then - echo "GPU VRAM drained (max used: $(( max_used / 1024 / 1024 )) MiB)" + if (( current_used <= threshold_bytes )); then + echo "GPU VRAM drained (max used: $(( current_used / 1024 / 1024 )) MiB)" + return 0 + fi + echo "GPU still holds $(( current_used / 1024 / 1024 )) MiB VRAM; waiting..." + sleep 5 + done + echo "WARNING: **************************************************************************************************************" >&2 + echo "WARNING: GPU(s) DID NOT DRAIN COMPLETELY " >&2 + echo "WARNING: GPU VRAM did not drain within ${timeout}s (max used: $(( current_used / 1024 / 1024 )) MiB); proceeding anyway" >&2 + echo "WARNING: **************************************************************************************************************" >&2 +} + +# Wait until all GPUs have dropped to 0% utilization, or until timeout. +# Call this after stop_server to confirm the GPU is truly idle before +# starting the next server (complements drain_gpu which checks VRAM). +# rocm-smi --showuse emits lines like: +# GPU[0] : GPU use (%): 12 +# timeout default: 60s +wait_gpu_idle() { + local timeout=${1:-60} + echo "--- :hourglass: waiting for GPU utilization to reach 0% (timeout ${timeout}s)" + local deadline + local max_util=0 + deadline=$(( $(date +%s) + timeout )) + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "rocm-smi unavailable; skipping GPU idle check" >&2 + return 0 + fi + while (( $(date +%s) < deadline )); do + + max_util=$(rocm-smi --showuse --noheader 2>/dev/null \ + | awk '/GPU use \(%\)/{if($NF+0>m)m=$NF+0} END{print m+0}') || true + if [[ -z "$max_util" ]]; then + echo "rocm-smi returned no data; skipping GPU idle check" >&2 + return 0 + fi + if (( max_util == 0 )); then + echo "GPU utilization is 0% on all devices" return 0 fi - echo "GPU still holds $(( max_used / 1024 / 1024 )) MiB VRAM; waiting..." + echo "GPU still at ${max_util}% utilization; waiting..." sleep 5 done - echo "WARNING: GPU VRAM did not drain within ${timeout}s (max used: $(( max_used / 1024 / 1024 )) MiB); proceeding anyway" >&2 + echo "WARNING: GPU utilization did not reach 0% within ${timeout}s (max: ${max_util}%); proceeding anyway" >&2 } \ No newline at end of file From 5dc69d0c93b16af93076cec1b908f52f5080f7cc Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Thu, 18 Jun 2026 09:12:44 -0500 Subject: [PATCH 37/62] updates for results storage Signed-off-by: Stacy Roberts --- lib/run.sh | 22 +++---- lib/run_vllm_bench.sh | 33 +++++++++- lib/server.sh | 3 +- workloads/gpt_oss_120b_mi355x_attn_sweep.yaml | 65 ++++++++++++++----- 4 files changed, 91 insertions(+), 32 deletions(-) diff --git a/lib/run.sh b/lib/run.sh index d16e412..384ea0b 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -60,19 +60,22 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" wait_healthy "$PORT" -#figuring out what the default attention backend is if [[ "$ATTN_BACKEND" == "default" ]]; then echo "--- :mag: attention backend selected by vLLM:" + _backend_lines="" if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then - grep -E "(Using|Overriding with|Incompatible).*(backend|Backend)" \ + _backend_lines=$(grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null \ - | grep "Worker_TP0\|Worker pid" \ - | sed 's/^/ /' || echo " (backend selection lines not found in log)" + | grep "rocm\.py") || true else - docker logs "$CONTAINER" 2>&1 \ - | grep -E "(Using|Overriding with|Incompatible).*(backend|Backend)" \ - | grep "Worker_TP0\|Worker pid" \ - | sed 's/^/ /' || echo " (backend selection lines not found in log)" + _backend_lines=$(docker logs "$CONTAINER" 2>&1 \ + | grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ + | grep "rocm\.py") || true + fi + if [[ -n "$_backend_lines" ]]; then + echo "$_backend_lines" | sed 's/^/ /' + else + echo " (backend selection lines not found in log)" fi fi @@ -95,9 +98,6 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do --model "$WORKLOAD_MODEL" \ --image "$WORKLOAD_IMAGE" \ --isl "$isl" --osl "$osl" --conc "$conc" || true - - echo "Giving GPU time to settle between tests" - wait_gpu_idle done <<< "$WORKLOAD_VLLM_BENCH_TSV" if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index 3c7fe7d..8cbe679 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -87,6 +87,9 @@ run_vllm_bench() { echo "--- :stopwatch: vllm bench serve ${name} (dataset=${dataset} isl=${input_len} osl=${output_len} conc=${max_concurrency} n=${num_prompts})" mkdir -p "$outdir" + local attn_backend="${ATTN_BACKEND:-default}" + local summary_file="${outdir}/bench-${name}-summary.txt" + local cmd=(vllm bench serve) [[ "$runtime" != "native" ]] && cmd=(docker exec "$container" "${cmd[@]}") @@ -138,10 +141,38 @@ run_vllm_bench() { cmd+=(--save-result --result-filename "$in_container_json") fi - "${cmd[@]}" + "${cmd[@]}" | tee "$summary_file" [[ "$runtime" != "native" ]] && docker cp "${container}:${in_container_json}" "$host_json" + # Resolve the actual attention backend selected by vLLM from the live server + # log (server is still running at this point). Named backends are already + # exact; only "default" needs resolution. + if [[ "$attn_backend" == "default" ]]; then + local _resolved + if [[ "$runtime" == "native" ]]; then + _resolved=$(grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null \ + | grep -m1 "rocm\.py" \ + | grep -oE "Using [A-Z_]+ backend" | awk '{print $2}') || true + else + _resolved=$(docker logs "$container" 2>&1 \ + | grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ + | grep -m1 "rocm\.py" \ + | grep -oE "Using [A-Z_]+ backend" | awk '{print $2}') || true + fi + [[ -n "$_resolved" ]] && attn_backend="$_resolved" + fi + + # Prepend context header to the summary file + local _tmp="${summary_file}.tmp" + { + echo "attention_backend: ${attn_backend}" + echo "isl: ${input_len} osl: ${output_len} conc: ${max_concurrency} n: ${num_prompts}" + echo "" + cat "$summary_file" + } > "$_tmp" && mv "$_tmp" "$summary_file" + python3 - "$host_json" "$num_prompts" <<'PY' import json, sys path, expected = sys.argv[1], int(sys.argv[2]) diff --git a/lib/server.sh b/lib/server.sh index ed93847..7ee6345 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -160,16 +160,15 @@ drain_gpu() { # timeout default: 60s wait_gpu_idle() { local timeout=${1:-60} + local max_util=0 echo "--- :hourglass: waiting for GPU utilization to reach 0% (timeout ${timeout}s)" local deadline - local max_util=0 deadline=$(( $(date +%s) + timeout )) if ! command -v rocm-smi >/dev/null 2>&1; then echo "rocm-smi unavailable; skipping GPU idle check" >&2 return 0 fi while (( $(date +%s) < deadline )); do - max_util=$(rocm-smi --showuse --noheader 2>/dev/null \ | awk '/GPU use \(%\)/{if($NF+0>m)m=$NF+0} END{print m+0}') || true if [[ -z "$max_util" ]]; then diff --git a/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml index 9cf4df1..80077ec 100644 --- a/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml +++ b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml @@ -14,30 +14,59 @@ vllm: --trust-remote-code --kv-cache-dtype=fp8 attention_backends: + - default - ROCM_AITER_UNIFIED_ATTN - ROCM_AITER_FA + - TRITON_ATTN -lm_eval: - model_args: - tokenized_requests: false - tokenizer_backend: null - timeout: 6000 - tasks: - - name: gsm8k - num_fewshot: 5 - model_args: - num_concurrent: 64 - max_length: 32768 - max_gen_toks: 8192 +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 vllm_bench: configs: - - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + - name: isl1024-osl1024-conc4 + dataset: random input_len: 1024 output_len: 1024 - num_prompts: 5 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy From 226dcd70e758c4536ec8046b0c560b8694fcc56a Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Fri, 19 Jun 2026 13:41:19 +0000 Subject: [PATCH 38/62] updates to summary output. gen_report for html generation. yamls for running on various models (not all complete yet) Signed-off-by: Stacy Roberts --- gen_report.py | 456 ++++++++++++++++++ lib/parse_workload.py | 2 +- lib/run_vllm_bench.sh | 22 +- .../attn_sweep_amd_gpt_oss_120b_mi355x.yaml | 74 +++ .../attn_sweep_deepseek_r1_0528_mi355x.yaml | 58 +++ .../attn_sweep_deepseek_r1_mxfp4_mi355x.yaml | 88 ++++ workloads/attn_sweep_gpt_oss_120b_mi355x.yaml | 74 +++ ..._sweep_triton_deepseek_r1_0528_mi355x.yaml | 105 ++++ workloads/deepseek_r1_0528_gsm8k_mi355x.yaml | 91 ++++ ...eek_r1_0528_tritonattnbackends_mi355x.yaml | 105 ++++ 10 files changed, 1067 insertions(+), 8 deletions(-) create mode 100644 gen_report.py create mode 100644 workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml create mode 100644 workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml create mode 100644 workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml create mode 100644 workloads/attn_sweep_gpt_oss_120b_mi355x.yaml create mode 100644 workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml create mode 100644 workloads/deepseek_r1_0528_gsm8k_mi355x.yaml create mode 100644 workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml diff --git a/gen_report.py b/gen_report.py new file mode 100644 index 0000000..48f6d96 --- /dev/null +++ b/gen_report.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""Generate one HTML benchmark report per model directory under ./results.""" + +import os +import re +import sys +import json +from pathlib import Path +from collections import defaultdict + +RESULTS_DIR = Path(__file__).parent / "results" +OUT_DIR = Path(__file__).parent + +# ── Metric extraction ───────────────────────────────────────────────────────── + +METRIC_PATTERNS = { + "Request throughput": r"Request throughput \(req/s\):\s+([\d.]+)", + "Output token throughput": r"Output token throughput \(tok/s\):\s+([\d.]+)", + "Total token throughput": r"Total token throughput \(tok/s\):\s+([\d.]+)", + "Benchmark duration": r"Benchmark duration \(s\):\s+([\d.]+)", + "Mean TTFT": r"Mean TTFT \(ms\):\s+([\d.]+)", + "Median TTFT": r"Median TTFT \(ms\):\s+([\d.]+)", + "P99 TTFT": r"P99 TTFT \(ms\):\s+([\d.]+)", + "Mean TPOT": r"Mean TPOT \(ms\):\s+([\d.]+)", + "Median TPOT": r"Median TPOT \(ms\):\s+([\d.]+)", + "P99 TPOT": r"P99 TPOT \(ms\):\s+([\d.]+)", + "Mean ITL": r"Mean ITL \(ms\):\s+([\d.]+)", + "Median ITL": r"Median ITL \(ms\):\s+([\d.]+)", + "P99 ITL": r"P99 ITL \(ms\):\s+([\d.]+)", +} + +HEADER_PATTERN = re.compile( + r"attention_backend:\s*(\S+).*?isl:\s*(\d+)\s+osl:\s*(\d+)\s+conc:\s*(\d+)", + re.DOTALL, +) +OVERRIDE_PATTERN = re.compile(r"Overriding with (\S+) out of potential backends") + + +def parse_txt(path: Path) -> dict | None: + text = path.read_text(errors="replace") + m = HEADER_PATTERN.search(text) + if not m: + return None + backend, isl, osl, conc = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4)) + override = OVERRIDE_PATTERN.search(text) + if override: + backend = override.group(1) + metrics = {} + for name, pat in METRIC_PATTERNS.items(): + hit = re.search(pat, text) + metrics[name] = float(hit.group(1)) if hit else None + return {"backend": backend, "isl": isl, "osl": osl, "conc": conc, "metrics": metrics} + + +# ── Directory scan ──────────────────────────────────────────────────────────── + +def collect_model_dir(model_dir: Path) -> dict: + """Return {backend -> {(isl,osl,conc) -> metrics_dict}}""" + data = defaultdict(dict) + + def ingest(txt_path: Path, backend_override: str | None = None): + rec = parse_txt(txt_path) + if rec is None: + return + backend = backend_override or rec["backend"] + key = (rec["isl"], rec["osl"], rec["conc"]) + data[backend][key] = rec["metrics"] + + # top-level summary files (backend comes from file content) + for txt in sorted(model_dir.glob("*-summary.txt")): + ingest(txt) + + # attn-BACKEND subdirs + for sub in sorted(model_dir.iterdir()): + if sub.is_dir() and sub.name.startswith("attn-"): + backend_name = sub.name[len("attn-"):] + for txt in sorted(sub.glob("*-summary.txt")): + ingest(txt, backend_override=backend_name) + + return dict(data) + + +# ── HTML generation ─────────────────────────────────────────────────────────── + +CSS = """ + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + + body { + font-family: ui-monospace, "Cascadia Code", "SF Mono", Menlo, Consolas, monospace; + background: #0f1117; + color: #e2e8f0; + padding: 2rem; + font-size: 14px; + line-height: 1.5; + } + + h1 { + font-size: 1.35rem; + font-weight: 600; + color: #f8fafc; + margin-bottom: 0.3rem; + letter-spacing: -0.01em; + } + + .subtitle { + color: #64748b; + font-size: 0.82rem; + margin-bottom: 1.75rem; + } + + .legend { + display: flex; + gap: 1.5rem; + margin-bottom: 1.5rem; + font-size: 0.75rem; + color: #64748b; + align-items: center; + } + .legend-item { display: flex; align-items: center; gap: 0.4rem; } + .swatch { width: 10px; height: 10px; border-radius: 2px; } + .swatch.green { background: #22c55e; } + .swatch.red { background: #ef4444; } + + .tab-bar { + display: flex; + gap: 0; + margin-bottom: 1.5rem; + border-bottom: 1px solid #1e293b; + } + .tab { + padding: 0.5rem 1.25rem; + font-size: 0.8rem; + cursor: pointer; + color: #64748b; + border-bottom: 2px solid transparent; + margin-bottom: -1px; + user-select: none; + transition: color 0.15s; + } + .tab:hover { color: #94a3b8; } + .tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } + .tab-content { display: none; } + .tab-content.active { display: block; } + + .conc-group { margin-bottom: 2.25rem; } + .conc-label { + font-size: 0.7rem; + font-weight: 700; + color: #475569; + text-transform: uppercase; + letter-spacing: 0.1em; + margin-bottom: 0.6rem; + } + + table { width: 100%; border-collapse: collapse; } + colgroup col:first-child { width: 13rem; } + + th, td { padding: 0.48rem 0.9rem; text-align: right; } + th:first-child, td:first-child { text-align: left; } + + thead tr { border-bottom: 1px solid #1e293b; } + thead th { + font-size: 0.72rem; + font-weight: 500; + color: #64748b; + text-transform: uppercase; + letter-spacing: 0.05em; + padding-bottom: 0.6rem; + } + thead th.backend-name { + font-size: 0.82rem; + text-transform: none; + letter-spacing: normal; + color: #cbd5e1; + font-weight: 600; + } + thead th.backend-name.default-backend { color: #38bdf8; } + + tr.section-row td { + font-size: 0.65rem; + text-transform: uppercase; + letter-spacing: 0.08em; + color: #334155; + padding-top: 0.9rem; + padding-bottom: 0.25rem; + border-bottom: none; + } + + tbody tr { border-bottom: 1px solid #0f172a; } + tbody tr:last-child { border-bottom: none; } + tbody tr:not(.section-row):hover { background: #131c2e; } + + td.metric-label { color: #94a3b8; font-size: 0.8rem; padding-left: 1.5rem; } + td.val { color: #cbd5e1; } + td.val .unit { color: #334155; font-size: 0.68rem; margin-left: 0.2rem; } + td.val.best { color: #22c55e; font-weight: 700; } + td.val.worst { color: #ef4444; } + + .delta { font-size: 0.68rem; margin-left: 0.3rem; opacity: 0.85; } + .delta.good { color: #22c55e; } + .delta.bad { color: #ef4444; } + .delta.neut { color: #475569; } +""" + +JS_TEMPLATE = """ +var BACKENDS = {backends_json}; + +var METRIC_DEFS = [ + {{ label: 'Request throughput', hi: true, unit: 'req/s' }}, + {{ label: 'Output token throughput', hi: true, unit: 'tok/s' }}, + {{ label: 'Total token throughput', hi: true, unit: 'tok/s' }}, + {{ label: 'Benchmark duration', hi: false, unit: 's' }}, +]; +var TTFT_DEFS = [ + {{ label: 'Mean TTFT', hi: false, unit: 'ms' }}, + {{ label: 'Median TTFT', hi: false, unit: 'ms' }}, + {{ label: 'P99 TTFT', hi: false, unit: 'ms' }}, +]; +var TPOT_DEFS = [ + {{ label: 'Mean TPOT', hi: false, unit: 'ms' }}, + {{ label: 'Median TPOT', hi: false, unit: 'ms' }}, + {{ label: 'P99 TPOT', hi: false, unit: 'ms' }}, +]; +var ITL_DEFS = [ + {{ label: 'Mean ITL', hi: false, unit: 'ms' }}, + {{ label: 'Median ITL', hi: false, unit: 'ms' }}, + {{ label: 'P99 ITL', hi: false, unit: 'ms' }}, +]; + +var DATA = {data_json}; + +function fmt(v, unit) {{ + if (v === null || v === undefined) return '—'; + if (unit === 'ms' || unit === 's' || unit === 'req/s' || unit === 'tok/s') return v.toFixed(2); + return String(v); +}} + +function deltaHtml(base, val, hi) {{ + if (base === null || val === null) return ''; + var pct = (val - base) / base * 100; + if (Math.abs(pct) < 0.1) return '(~0%)'; + var sign = pct > 0 ? '+' : ''; + var cls = hi ? (pct > 0 ? 'good' : 'bad') : (pct < 0 ? 'good' : 'bad'); + return '(' + sign + pct.toFixed(1) + '%)'; +}} + +function buildTable(key, concLabel) {{ + var rows = DATA[key]; + if (!rows) return '

No data for ' + key + '

'; + + var sections = [ + {{ label: 'Throughput', defs: METRIC_DEFS }}, + {{ label: 'Time to First Token', defs: TTFT_DEFS }}, + {{ label: 'Time per Output Token', defs: TPOT_DEFS }}, + {{ label: 'Inter-token Latency', defs: ITL_DEFS }}, + ]; + + var headCols = 'Metric'; + BACKENDS.forEach(function(b) {{ + var cls = 'backend-name' + (b.isDefault ? ' default-backend' : ''); + var tag = b.isDefault ? ' (default)' : ''; + headCols += '' + b.label + tag + ''; + }}); + + var bodyHtml = ''; + sections.forEach(function(sec) {{ + bodyHtml += '' + sec.label + ''; + sec.defs.forEach(function(def) {{ + var vals = rows[def.label]; + if (!vals) return; + var valid = vals.map(function(v, i) {{ return {{ v: v, i: i }}; }}).filter(function(x) {{ return x.v !== null; }}); + var bestIdx = null, worstIdx = null; + if (valid.length > 1) {{ + var sorted = valid.slice().sort(function(a, b) {{ return def.hi ? b.v - a.v : a.v - b.v; }}); + bestIdx = sorted[0].i; + worstIdx = sorted[sorted.length - 1].i; + }} + var baseVal = vals[0]; + var cells = '' + def.label + ''; + vals.forEach(function(v, i) {{ + var cls = 'val'; + if (i === bestIdx) cls += ' best'; + if (i === worstIdx) cls += ' worst'; + var delta = i === 0 ? '' : deltaHtml(baseVal, v, def.hi); + cells += '' + fmt(v, def.unit) + '' + def.unit + '' + delta + ''; + }}); + bodyHtml += '' + cells + ''; + }}); + }}); + + return '' + BACKENDS.map(function() {{ return ''; }}).join('') + '' + + '' + headCols + '' + + '' + bodyHtml + '
'; +}} + +function buildContent(islKey) {{ + var concKeys = Object.keys(DATA).filter(function(k) {{ return k.startsWith(islKey + '-'); }}); + concKeys.sort(function(a, b) {{ + var ca = parseInt(a.replace(/.*conc/, '')); + var cb = parseInt(b.replace(/.*conc/, '')); + return ca - cb; + }}); + var html = ''; + concKeys.forEach(function(k) {{ + var conc = k.replace(/.*conc/, ''); + html += '
Concurrency ' + conc + '
' + buildTable(k) + '
'; + }}); + return html || '

No data

'; +}} + +function switchTab(el) {{ + document.querySelectorAll('.tab').forEach(function(t) {{ t.classList.remove('active'); }}); + document.querySelectorAll('.tab-content').forEach(function(c) {{ c.classList.remove('active'); }}); + el.classList.add('active'); + var target = document.getElementById(el.dataset.tab); + target.classList.add('active'); + if (!target.dataset.built) {{ + target.innerHTML = buildContent(el.dataset.tab); + target.dataset.built = '1'; + }} +}} + +// Build first tab on load +(function() {{ + var firstTab = document.querySelector('.tab.active'); + if (firstTab) {{ + var target = document.getElementById(firstTab.dataset.tab); + if (target) {{ + target.innerHTML = buildContent(firstTab.dataset.tab); + target.dataset.built = '1'; + }} + }} +}})(); +""" + + +def build_html(model_name: str, backend_data: dict) -> str: + # Determine backends — pick a stable default order: + # prefer ROCM_AITER_UNIFIED_ATTN as default; otherwise alphabetical first + all_backends = sorted(backend_data.keys()) + preferred_default = "ROCM_AITER_UNIFIED_ATTN" + if preferred_default in all_backends: + all_backends.remove(preferred_default) + all_backends = [preferred_default] + all_backends + + # Collect all (isl,osl,conc) test points + all_keys: set[tuple] = set() + for bdata in backend_data.values(): + all_keys.update(bdata.keys()) + + # Build DATA dict: { "isl1024-conc4": { "Mean TTFT": [v_backend0, v_backend1, ...], ... } } + data_dict = {} + for (isl, osl, conc) in sorted(all_keys): + dk = f"isl{isl}-conc{conc}" + row: dict[str, list] = {} + for mname in METRIC_PATTERNS: + vals = [] + for b in all_backends: + bdata = backend_data.get(b, {}) + point = bdata.get((isl, osl, conc), {}) + vals.append(point.get(mname) if point else None) + row[mname] = vals + data_dict[dk] = row + + # Determine ISL groups for tabs + isl_values = sorted({k[0] for k in all_keys}) + osl_values = sorted({k[1] for k in all_keys}) + osl_label = "/".join(str(o) for o in osl_values) + + backends_js = json.dumps([ + {"key": b.lower().replace("_", ""), "label": b, "isDefault": i == 0} + for i, b in enumerate(all_backends) + ], indent=2) + + data_js = json.dumps(data_dict, indent=2) + + # Tab bar HTML + tabs_html = "" + tab_divs = "" + for i, isl in enumerate(isl_values): + tab_id = f"isl{isl}" + active = " active" if i == 0 else "" + tabs_html += f'
ISL {isl} / OSL {osl_label}
\n' + tab_divs += f'
\n' + + subtitle = f"Model: {model_name}" + + html = f""" + + + + + Benchmark — {model_name} + + + + +

Attention Backend Benchmark

+

{subtitle}

+ +
+
Best in row
+
Worst in row
+ · deltas relative to default (first) backend +
+ +
+{tabs_html}
+ +{tab_divs} + + + +""" + return html + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def model_name_from_dir(dir_name: str) -> str: + """Best-effort human label from directory name like attn_sweep_gpt_oss_120b_mi355x.""" + # strip leading attn_sweep_ or similar prefix + name = re.sub(r"^attn_sweep_", "", dir_name) + return name + + +def main(): + if not RESULTS_DIR.exists(): + sys.exit(f"results/ not found at {RESULTS_DIR}") + + model_dirs = [d for d in sorted(RESULTS_DIR.iterdir()) if d.is_dir()] + if not model_dirs: + sys.exit("No subdirectories found in results/") + + for model_dir in model_dirs: + backend_data = collect_model_dir(model_dir) + if not backend_data: + print(f" skip {model_dir.name} — no parseable summary files") + continue + + model_name = model_name_from_dir(model_dir.name) + html = build_html(model_name, backend_data) + + out_path = OUT_DIR / f"benchmark-{model_dir.name}.html" + out_path.write_text(html) + backends = list(backend_data.keys()) + points = sum(len(v) for v in backend_data.values()) + print(f" wrote {out_path.name} ({len(backends)} backends, {points} test points)") + + +if __name__ == "__main__": + main() diff --git a/lib/parse_workload.py b/lib/parse_workload.py index e2eb900..2594694 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -26,7 +26,7 @@ KNOWN_ATTENTION_BACKENDS = { "default", "FLASH_ATTN", "FLASHINFER", "XFORMERS", "TRITON_ATTN", - "TRITON_MLA", "ROCM_FLASH", "PAGED_ATTENTION", "ROCM_AITER_FA", + "TRITON_MLA", "ROCM_FLASH", "PAGED_ATTENTION", "ROCM_AITER_FA", "ROCM_AITER_MHA", "ROCM_AITER_UNIFIED_ATTN", "ROCM_ATTN", "ROCM_AITER_MLA", "ROCM_AITER_MLA_SPARSE", "ROCM_AITER_TRITON_MLA" } diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index 8cbe679..3a4fa77 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -150,15 +150,23 @@ run_vllm_bench() { # exact; only "default" needs resolution. if [[ "$attn_backend" == "default" ]]; then local _resolved + local _log_lines if [[ "$runtime" == "native" ]]; then - _resolved=$(grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ - "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null \ - | grep -m1 "rocm\.py" \ - | grep -oE "Using [A-Z_]+ backend" | awk '{print $2}') || true + _log_lines=$(grep -E "(Overriding with|Using [A-Z_]+ backend)" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null) || true else - _resolved=$(docker logs "$container" 2>&1 \ - | grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ - | grep -m1 "rocm\.py" \ + _log_lines=$(docker logs "$container" 2>&1 \ + | grep -E "(Overriding with|Using [A-Z_]+ backend)") || true + fi + # Prefer "Overriding with X" — this is the definitive selection line when + # vLLM discards incompatible backends and picks the actual one used. + _resolved=$(echo "$_log_lines" \ + | grep "Overriding with" | head -1 \ + | grep -oE "Overriding with [A-Z_]+" | awk '{print $3}') || true + # Fall back to "Using X backend" if no override line present + if [[ -z "$_resolved" ]]; then + _resolved=$(echo "$_log_lines" \ + | grep -m1 "Using [A-Z_]+ backend" \ | grep -oE "Using [A-Z_]+ backend" | awk '{print $2}') || true fi [[ -n "$_resolved" ]] && attn_backend="$_resolved" diff --git a/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..b35fcb0 --- /dev/null +++ b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,74 @@ +# GPT-OSS 120B on MI355X +name: attn_sweep_amd_gpt_oss_120b_w_mxfp4_a_fp8_mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: amd/gpt-oss-120b-w-mxfp4-a-fp8 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + attention_backends: + - default + - TRITON_ATTN + #- ROCM_AITER_FA default 2026-06-18 + #- ROCM_ATTN #fails for attention sinks not supported + - ROCM_AITER_UNIFIED_ATTN + + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..b1f9a95 --- /dev/null +++ b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,58 @@ +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: attn-sweep-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + - default + - ROCM_AITER_MLA #default 2026-06-18 + - ROCM_AITER_TRITON_MLA + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 \ No newline at end of file diff --git a/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..2ee3bf9 --- /dev/null +++ b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,88 @@ +# Attention Backend sweep +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# +name: attn-sweep-deepseek-r1-mxfp4-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false,},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} +# adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..d8a81fc --- /dev/null +++ b/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,74 @@ +# GPT-OSS 120B on MI355X +name: attn_sweep_gpt_oss_120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + attention_backends: + - default + #- ROCM_ATTN #attention sinks not supported + - TRITON_ATTN + #- ROCM_AITER_UNIFIED_ATTN this one is the default + #- ROCM_AITER_FA #fails "attention sinks not supported" + + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..46b3d89 --- /dev/null +++ b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,105 @@ +# +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: attn-sweep-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 16 + --async-scheduling + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + - default + - TRITON_MLA + +# lm_eval: +# model_args: +# tokenized_requests: false +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/deepseek_r1_0528_gsm8k_mi355x.yaml b/workloads/deepseek_r1_0528_gsm8k_mi355x.yaml new file mode 100644 index 0000000..c7af18d --- /dev/null +++ b/workloads/deepseek_r1_0528_gsm8k_mi355x.yaml @@ -0,0 +1,91 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# max_num_seqs=32 (much lower than other groups — intentional per original config). +# extra_args: async-scheduling, kv-cache-dtype=fp8, block-size=1, compilation-config +# The compilation-config JSON is passed as a single quoted string in serve_args. +# No ACCURACY group for this model in vllm_upstream.yaml. +name: deepseek-r1-mxfp4-upstream-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false,},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} +# adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml b/workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml new file mode 100644 index 0000000..97b7529 --- /dev/null +++ b/workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml @@ -0,0 +1,105 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: upstream-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 16 + --async-scheduling + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + - default + - TRITON_MLA #Block size of 16 + +# lm_eval: +# model_args: +# tokenized_requests: false +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl1024-osl1024-conc64 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 64 + # - name: isl8192-osl1024-conc4 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl8192-osl1024-conc64 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 64 From deebe2b672a27d32e289ea662268ba5127b1483a Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Fri, 19 Jun 2026 16:50:02 +0000 Subject: [PATCH 39/62] selects correct default backend for marker Signed-off-by: Stacy Roberts --- gen_report.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/gen_report.py b/gen_report.py index 48f6d96..b933cfd 100644 --- a/gen_report.py +++ b/gen_report.py @@ -54,30 +54,38 @@ def parse_txt(path: Path) -> dict | None: # ── Directory scan ──────────────────────────────────────────────────────────── -def collect_model_dir(model_dir: Path) -> dict: - """Return {backend -> {(isl,osl,conc) -> metrics_dict}}""" +def collect_model_dir(model_dir: Path) -> tuple[dict, str | None]: + """Return ({backend -> {(isl,osl,conc) -> metrics_dict}}, default_backend_name) + + The default backend is the one whose results live at the top level of the + model directory (not inside an attn-* subdirectory). + """ data = defaultdict(dict) + default_backend: str | None = None - def ingest(txt_path: Path, backend_override: str | None = None): + def ingest(txt_path: Path, backend_override: str | None = None) -> str | None: rec = parse_txt(txt_path) if rec is None: - return + return None backend = backend_override or rec["backend"] key = (rec["isl"], rec["osl"], rec["conc"]) data[backend][key] = rec["metrics"] + return backend - # top-level summary files (backend comes from file content) + # top-level summary files — these are the default backend runs for txt in sorted(model_dir.glob("*-summary.txt")): - ingest(txt) + name = ingest(txt) + if name and default_backend is None: + default_backend = name - # attn-BACKEND subdirs + # attn-BACKEND subdirs — non-default backends for sub in sorted(model_dir.iterdir()): if sub.is_dir() and sub.name.startswith("attn-"): backend_name = sub.name[len("attn-"):] for txt in sorted(sub.glob("*-summary.txt")): ingest(txt, backend_override=backend_name) - return dict(data) + return dict(data), default_backend # ── HTML generation ─────────────────────────────────────────────────────────── @@ -334,14 +342,12 @@ def ingest(txt_path: Path, backend_override: str | None = None): """ -def build_html(model_name: str, backend_data: dict) -> str: - # Determine backends — pick a stable default order: - # prefer ROCM_AITER_UNIFIED_ATTN as default; otherwise alphabetical first +def build_html(model_name: str, backend_data: dict, default_backend: str | None) -> str: + # Put the default backend first; sort the rest alphabetically all_backends = sorted(backend_data.keys()) - preferred_default = "ROCM_AITER_UNIFIED_ATTN" - if preferred_default in all_backends: - all_backends.remove(preferred_default) - all_backends = [preferred_default] + all_backends + if default_backend and default_backend in all_backends: + all_backends.remove(default_backend) + all_backends = [default_backend] + all_backends # Collect all (isl,osl,conc) test points all_keys: set[tuple] = set() @@ -437,13 +443,13 @@ def main(): sys.exit("No subdirectories found in results/") for model_dir in model_dirs: - backend_data = collect_model_dir(model_dir) + backend_data, default_backend = collect_model_dir(model_dir) if not backend_data: print(f" skip {model_dir.name} — no parseable summary files") continue model_name = model_name_from_dir(model_dir.name) - html = build_html(model_name, backend_data) + html = build_html(model_name, backend_data, default_backend) out_path = OUT_DIR / f"benchmark-{model_dir.name}.html" out_path.write_text(html) From 9183497bf6ec1e7377ace6686092e702464a85f4 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Fri, 19 Jun 2026 17:46:23 +0000 Subject: [PATCH 40/62] more attention sweep yamls Signed-off-by: Stacy Roberts --- workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml | 2 +- workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml index b1f9a95..9762139 100644 --- a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml +++ b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml @@ -15,7 +15,7 @@ vllm: --dtype auto --max-num-batched-tokens 131072 --max-num-seqs 1024 - --gpu-memory-utilization 0.95 + --gpu-memory-utilization 0.92 --max-model-len 10240 --block-size 1 --async-scheduling diff --git a/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml index 46b3d89..021a56d 100644 --- a/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml +++ b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml @@ -27,7 +27,7 @@ vllm: USE_FASTSAFETENSOR: 1 SAFETENSORS_FAST_GPU: 1 attention_backends: - - default + #- default - TRITON_MLA # lm_eval: From ac9238379e7dff29ce2378822e6f43dfdb5343c1 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Mon, 22 Jun 2026 15:49:27 +0000 Subject: [PATCH 41/62] added recovery from failure. gen_report.py addes header file for easier viewing. All current attn_sweep yamls updated Signed-off-by: Stacy Roberts --- README.md | 17 +- gen_report.py | 152 +++++++++++++++++- lib/run.sh | 38 +++-- .../attn_sweep_amd_gpt_oss_120b_mi355x.yaml | 2 +- .../attn_sweep_deepseek_r1_0528_mi355x.yaml | 2 +- .../attn_sweep_deepseek_r1_mxfp4_mi355x.yaml | 11 +- workloads/attn_sweep_gpt_oss_120b_mi355x.yaml | 2 +- ...sweep_triton_deepseek_r1_mxfp4_mi355x.yaml | 106 ++++++++++++ 8 files changed, 307 insertions(+), 23 deletions(-) create mode 100644 workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml diff --git a/README.md b/README.md index 6e729fb..166d691 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Each recipe is one `(model, hardware, set of tasks)` combination. The Buildkite workloads/ one YAML per (model, hardware) recipe lib/ orchestrator (run.sh), helpers, GPU profiles .buildkite/ pipeline bootstrap and step generator +gen_report.py generate HTML benchmark reports from results/ CLAUDE.md agent conventions and detailed Buildkite workflow ``` @@ -84,7 +85,7 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: -- **`vllm.attention_backends`** is an optional list of vLLM attention backend names (`FLASH_ATTN`, `FLASHINFER`, `XFORMERS`, `TRITON_ATTN`, `TRITON_MLA`, `ROCM_FLASH`, `PAGED_ATTENTION`,`ROCM_AITER_FA`,`ROCM_AITER_UNIFIED_ATTN`, `ROCM_ATTN`,`ROCM_AITER_MLA`,`ROCM_AITER_MLA_SPARSE`, `ROCM_AITER_TRITON_MLA`). When set, the orchestrator starts the server once per backend — adding `--attention-backend $ATTN_BACKEND` — and runs the complete eval suite (bench, lm_eval, bfcl) for each. Results are stored under `results//attn-/` so every backend gets its own isolated output directory. Without this field, the server starts once with whatever attention backend vLLM selects by default and results go to `results//` as usual. See `workloads/gpt_oss_120b_mi355x_attn_sweep.yaml` for an example. +- **`vllm.attention_backends`** is an optional list of vLLM attention backend names (`FLASH_ATTN`, `FLASHINFER`, `XFORMERS`, `TRITON_ATTN`, `TRITON_MLA`, `ROCM_FLASH`, `PAGED_ATTENTION`,`ROCM_AITER_FA`,`ROCM_AITER_UNIFIED_ATTN`, `ROCM_ATTN`,`ROCM_AITER_MLA`,`ROCM_AITER_MLA_SPARSE`, `ROCM_AITER_TRITON_MLA`). When set, the orchestrator starts the server once per backend — adding `--attention-backend $ATTN_BACKEND` — and runs the complete eval suite (bench, lm_eval, bfcl) for each. Results are stored under `results//attn-/` so every backend gets its own isolated output directory. Without this field, the server starts once with whatever attention backend vLLM selects by default and results go to `results//` as usual. See `workloads/attn-sweep-gpt-oss-120b-mi355x.yaml` for an example. - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. @@ -135,6 +136,20 @@ A real run needs a GPU host with Docker, vLLM, and lm-eval available: Locally, you can smoke-test recipe changes without a GPU — see `CLAUDE.md` for the parser stub and shell-syntax checks. +## Benchmark reports + +After a run completes, generate interactive HTML reports from the `results/` directory: + +```bash +python3 gen_report.py +``` + +This writes one `benchmark-.html` per model directory found under `results/`, plus a `benchmark-index.html` wrapper. Open `benchmark-index.html` in a browser to tab between all models in one page — each model's report loads on demand when its tab is clicked. + +If reports from previous rounds are already present in the directory, `benchmark-index.html` will include them alongside any newly generated ones, so the index always covers every available model regardless of which models were in the current run. + +Each per-model report shows attention backend results side by side, with tabs for each input sequence length, color-coded best/worst values per metric, and percentage deltas relative to the default backend. + ## Agents `CLAUDE.md` has conventions for AI agents working in this repo: smoke-testing changes, launching Buildkite builds for a chosen branch/commit, and the AI-assistance disclosure rule for PRs and commits. diff --git a/gen_report.py b/gen_report.py index b933cfd..0e739b4 100644 --- a/gen_report.py +++ b/gen_report.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Generate one HTML benchmark report per model directory under ./results.""" +"""Generate one HTML benchmark report per model directory under ./results, +plus a benchmark-index.html wrapper for tabbing between models.""" import os import re @@ -425,11 +426,137 @@ def build_html(model_name: str, backend_data: dict, default_backend: str | None) return html +# ── Index wrapper ───────────────────────────────────────────────────────────── + +INDEX_CSS = """ + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + + html, body { height: 100%; } + + body { + font-family: ui-monospace, "Cascadia Code", "SF Mono", Menlo, Consolas, monospace; + background: #0f1117; + color: #e2e8f0; + font-size: 14px; + display: flex; + flex-direction: column; + } + + .header { + padding: 0.75rem 2rem 0; + flex-shrink: 0; + } + + h1 { + font-size: 1.2rem; + font-weight: 600; + color: #f8fafc; + margin-bottom: 0.6rem; + letter-spacing: -0.01em; + } + + .model-tab-bar { + display: flex; + gap: 0; + border-bottom: 1px solid #1e293b; + } + + .model-tab { + padding: 0.5rem 1.4rem; + font-size: 0.8rem; + cursor: pointer; + color: #64748b; + border-bottom: 2px solid transparent; + margin-bottom: -1px; + user-select: none; + white-space: nowrap; + transition: color 0.15s; + } + .model-tab:hover { color: #94a3b8; } + .model-tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } + + .frame-container { + flex: 1; + position: relative; + } + + iframe { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + border: none; + display: none; + } + iframe.active { display: block; } +""" + +INDEX_JS = """ +function switchModel(el) { + document.querySelectorAll('.model-tab').forEach(function(t) { t.classList.remove('active'); }); + document.querySelectorAll('iframe').forEach(function(f) { f.classList.remove('active'); }); + el.classList.add('active'); + var frame = document.getElementById(el.dataset.frame); + frame.classList.add('active'); + // lazy-load: only set src when first activated + if (!frame.src || frame.src === 'about:blank') { + frame.src = frame.dataset.src; + } +} + +// activate first tab on load +(function() { + var first = document.querySelector('.model-tab'); + if (first) switchModel(first); +})(); +""" + + +def build_index_html(reports: list[tuple[str, str]]) -> str: + """Build a single-page wrapper that tabs between per-model report iframes. + + reports: list of (model_label, filename) sorted as desired. + """ + tabs_html = "" + frames_html = "" + for i, (label, filename) in enumerate(reports): + frame_id = f"frame-{i}" + tabs_html += f'
{label}
\n' + frames_html += f' \n' + + return f""" + + + + + Benchmark Reports + + + + +
+

Attention Backend Benchmarks

+
+{tabs_html}
+
+ +
+{frames_html}
+ + + + +""" + + # ── Main ────────────────────────────────────────────────────────────────────── def model_name_from_dir(dir_name: str) -> str: """Best-effort human label from directory name like attn_sweep_gpt_oss_120b_mi355x.""" - # strip leading attn_sweep_ or similar prefix name = re.sub(r"^attn_sweep_", "", dir_name) return name @@ -442,6 +569,8 @@ def main(): if not model_dirs: sys.exit("No subdirectories found in results/") + reports: list[tuple[str, str]] = [] + for model_dir in model_dirs: backend_data, default_backend = collect_model_dir(model_dir) if not backend_data: @@ -451,11 +580,26 @@ def main(): model_name = model_name_from_dir(model_dir.name) html = build_html(model_name, backend_data, default_backend) - out_path = OUT_DIR / f"benchmark-{model_dir.name}.html" + out_filename = f"benchmark-{model_dir.name}.html" + out_path = OUT_DIR / out_filename out_path.write_text(html) backends = list(backend_data.keys()) points = sum(len(v) for v in backend_data.values()) - print(f" wrote {out_path.name} ({len(backends)} backends, {points} test points)") + print(f" wrote {out_filename} ({len(backends)} backends, {points} test points)") + reports.append((model_name, out_filename)) + + # Also pick up any pre-existing benchmark-*.html files not produced this run + # (e.g. from a previous round with different models), so the index is complete. + existing = {fname for _, fname in reports} + for html_path in sorted(OUT_DIR.glob("benchmark-*.html")): + if html_path.name not in existing and html_path.name != "benchmark-index.html": + label = model_name_from_dir(html_path.stem.removeprefix("benchmark-")) + reports.append((label, html_path.name)) + + if reports: + index_path = OUT_DIR / "benchmark-index.html" + index_path.write_text(build_index_html(reports)) + print(f" wrote benchmark-index.html ({len(reports)} model(s))") if __name__ == "__main__": diff --git a/lib/run.sh b/lib/run.sh index 384ea0b..8ed061e 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -56,21 +56,32 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do trap 'stop_server "$CONTAINER"' EXIT - start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ - "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" - wait_healthy "$PORT" + if ! start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME"; then + echo "^^^ +++ ERROR: start_server failed for backend ${ATTN_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if ! wait_healthy "$PORT"; then + echo "^^^ +++ ERROR: vLLM never became healthy for backend ${ATTN_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi if [[ "$ATTN_BACKEND" == "default" ]]; then echo "--- :mag: attention backend selected by vLLM:" _backend_lines="" if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then - _backend_lines=$(grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ - "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null \ - | grep "rocm\.py") || true + _backend_lines=$(grep -E "(Overriding with|Using [A-Z_]+ backend)" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null) || true else _backend_lines=$(docker logs "$CONTAINER" 2>&1 \ - | grep -E "\[(rocm|selector)\.py:[0-9]+\].*Using [A-Z_]+ backend" \ - | grep "rocm\.py") || true + | grep -E "(Overriding with|Using [A-Z_]+ backend)") || true fi if [[ -n "$_backend_lines" ]]; then echo "$_backend_lines" | sed 's/^/ /' @@ -85,10 +96,13 @@ for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do # perf dashboard ingest endpoint. while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do [[ -z "$bname" ]] && continue - run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ - "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ - "$conc" "$speed_subset" "$speed_category" \ - "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR" + if ! run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR"; then + echo "^^^ +++ ERROR: run_vllm_bench failed for ${bname} (backend ${ATTN_BACKEND}); skipping run" >&2 + continue + fi python3 "$DIR/ingest_perf.py" \ --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ diff --git a/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml index b35fcb0..dd46b4b 100644 --- a/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml +++ b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -1,5 +1,5 @@ # GPT-OSS 120B on MI355X -name: attn_sweep_amd_gpt_oss_120b_w_mxfp4_a_fp8_mi355x +name: attn-sweep-amd-gpt-oss-120b-w-mxfp4-a-fp8-mi355x gpu: MI355X num_gpus: 8 # Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x diff --git a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml index 9762139..a66dbd2 100644 --- a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml +++ b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml @@ -27,7 +27,7 @@ vllm: SAFETENSORS_FAST_GPU: 1 attention_backends: - default - - ROCM_AITER_MLA #default 2026-06-18 + #- ROCM_AITER_MLA #default 2026-06-18 - ROCM_AITER_TRITON_MLA vllm_bench: diff --git a/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml index 2ee3bf9..2bb32e0 100644 --- a/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml +++ b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml @@ -2,7 +2,7 @@ # SERVING only — TP=8, MXFP4 preview checkpoint # in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] # -name: attn-sweep-deepseek-r1-mxfp4-mi355x +name: attn-sweep-deepseek-r1-mxfp4-MI355X gpu: MI355X num_gpus: 8 nightly: false @@ -21,8 +21,13 @@ vllm: --max-model-len 10240 --block-size 1 --async-scheduling - --compilation-config {"pass_config":{"eliminate_noops":false,},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} -# adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + # adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + attention_backends: + - default + - ROCM_AITER_TRITON_MLA + - ROCM_AITER_MLA + vllm_bench: configs: diff --git a/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml index d8a81fc..051ef62 100644 --- a/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml +++ b/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml @@ -1,5 +1,5 @@ # GPT-OSS 120B on MI355X -name: attn_sweep_gpt_oss_120b-mi355x +name: attn-sweep-gpt-oss-120b-mi355x gpu: MI355X num_gpus: 8 # Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x diff --git a/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..2c2878c --- /dev/null +++ b/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,106 @@ +# +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: attn-sweep-deepseek-r1-mxfp4-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 16 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + #- default + - TRITON_MLA + +# lm_eval: +# model_args: +# tokenized_requests: false +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 From 7d97b59799487d6cba25f2f539f17231b24232f9 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Mon, 22 Jun 2026 20:47:01 +0000 Subject: [PATCH 42/62] update config Signed-off-by: Stacy Roberts --- workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml index a66dbd2..9bceecd 100644 --- a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml +++ b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml @@ -19,6 +19,7 @@ vllm: --max-model-len 10240 --block-size 1 --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} env: VLLM_ROCM_USE_AITER: 1 HSA_NO_SCRATCH_RECLAIM: 1 From d8ba38c8156bd3593208f601e7d1ebc6c2d93e73 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Tue, 23 Jun 2026 14:37:20 +0000 Subject: [PATCH 43/62] clean up messages on attn backends Signed-off-by: Stacy Roberts --- workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml | 1 + workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml | 1 + workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml | 2 +- workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml index 9bceecd..14cf236 100644 --- a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml +++ b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml @@ -30,6 +30,7 @@ vllm: - default #- ROCM_AITER_MLA #default 2026-06-18 - ROCM_AITER_TRITON_MLA + #- ROCM_AITER_MLA_SPARSE #not supported Reason: ['non-sparse not supported'] vllm_bench: configs: diff --git a/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml index 2bb32e0..cb80109 100644 --- a/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml +++ b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml @@ -27,6 +27,7 @@ vllm: - default - ROCM_AITER_TRITON_MLA - ROCM_AITER_MLA + #- ROCM_AITER_MLA_SPARSE #not supported Reason: ['non-sparse not supported'] vllm_bench: diff --git a/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml index 021a56d..ba9189a 100644 --- a/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml +++ b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml @@ -27,7 +27,7 @@ vllm: USE_FASTSAFETENSOR: 1 SAFETENSORS_FAST_GPU: 1 attention_backends: - #- default + #- default #no need to run is run in non-triton config - TRITON_MLA # lm_eval: diff --git a/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml index 2c2878c..707ae8d 100644 --- a/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml +++ b/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml @@ -28,7 +28,7 @@ vllm: USE_FASTSAFETENSOR: 1 SAFETENSORS_FAST_GPU: 1 attention_backends: - #- default + #- default #no need to run is run in non-triton config - TRITON_MLA # lm_eval: From f7e9e3a32eac9b0f3112fc9ef8d9b76b9fb01671 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Tue, 9 Jun 2026 20:25:10 +0530 Subject: [PATCH 44/62] [BFCL] Add multiturn test group Signed-off-by: Tarun Kumar Signed-off-by: Stacy Roberts --- workloads/deepseek_v4_flash_b200.yaml | 1 + workloads/deepseek_v4_pro_5_h200.yaml | 1 + workloads/glm_5_1_h200.yaml | 1 + workloads/gpt_oss_120b_h200.yaml | 1 + workloads/kimi_k2_5_h200.yaml | 1 + workloads/minimax_m2_5_h200.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index 694ab37..b0826b8 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -29,6 +29,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 lm_eval: diff --git a/workloads/deepseek_v4_pro_5_h200.yaml b/workloads/deepseek_v4_pro_5_h200.yaml index 27ef33d..a53d751 100644 --- a/workloads/deepseek_v4_pro_5_h200.yaml +++ b/workloads/deepseek_v4_pro_5_h200.yaml @@ -45,6 +45,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/glm_5_1_h200.yaml b/workloads/glm_5_1_h200.yaml index 90bacd8..148cd23 100644 --- a/workloads/glm_5_1_h200.yaml +++ b/workloads/glm_5_1_h200.yaml @@ -36,6 +36,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index 03975f3..31cf6d4 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -33,6 +33,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/kimi_k2_5_h200.yaml b/workloads/kimi_k2_5_h200.yaml index 684bcca..0f6ec46 100644 --- a/workloads/kimi_k2_5_h200.yaml +++ b/workloads/kimi_k2_5_h200.yaml @@ -33,6 +33,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/minimax_m2_5_h200.yaml b/workloads/minimax_m2_5_h200.yaml index 58f7349..c5dd892 100644 --- a/workloads/minimax_m2_5_h200.yaml +++ b/workloads/minimax_m2_5_h200.yaml @@ -36,6 +36,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: From cdacc2ee627769922a716ae4411a33f8e8885c97 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 10 Jun 2026 20:52:56 -0700 Subject: [PATCH 45/62] Revert "[BFCL] Add multiturn test group for tool accuracy" (#15) (#16) Co-authored-by: Claude Fable 5 Signed-off-by: Stacy Roberts --- workloads/deepseek_v4_flash_b200.yaml | 1 - workloads/deepseek_v4_pro_5_h200.yaml | 1 - workloads/glm_5_1_h200.yaml | 1 - workloads/gpt_oss_120b_h200.yaml | 1 - workloads/kimi_k2_5_h200.yaml | 1 - workloads/minimax_m2_5_h200.yaml | 1 - 6 files changed, 6 deletions(-) diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index b0826b8..694ab37 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -29,7 +29,6 @@ bfcl: - multiple - parallel - parallel_multiple - - multi_turn num_threads: 8 lm_eval: diff --git a/workloads/deepseek_v4_pro_5_h200.yaml b/workloads/deepseek_v4_pro_5_h200.yaml index a53d751..27ef33d 100644 --- a/workloads/deepseek_v4_pro_5_h200.yaml +++ b/workloads/deepseek_v4_pro_5_h200.yaml @@ -45,7 +45,6 @@ bfcl: - multiple - parallel - parallel_multiple - - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/glm_5_1_h200.yaml b/workloads/glm_5_1_h200.yaml index 148cd23..90bacd8 100644 --- a/workloads/glm_5_1_h200.yaml +++ b/workloads/glm_5_1_h200.yaml @@ -36,7 +36,6 @@ bfcl: - multiple - parallel - parallel_multiple - - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index 31cf6d4..03975f3 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -33,7 +33,6 @@ bfcl: - multiple - parallel - parallel_multiple - - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/kimi_k2_5_h200.yaml b/workloads/kimi_k2_5_h200.yaml index 0f6ec46..684bcca 100644 --- a/workloads/kimi_k2_5_h200.yaml +++ b/workloads/kimi_k2_5_h200.yaml @@ -33,7 +33,6 @@ bfcl: - multiple - parallel - parallel_multiple - - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/minimax_m2_5_h200.yaml b/workloads/minimax_m2_5_h200.yaml index c5dd892..58f7349 100644 --- a/workloads/minimax_m2_5_h200.yaml +++ b/workloads/minimax_m2_5_h200.yaml @@ -36,7 +36,6 @@ bfcl: - multiple - parallel - parallel_multiple - - multi_turn num_threads: 8 vllm_bench: From e4864c8b12ae35a45a045ec944b891095c4fb0b9 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Fri, 12 Jun 2026 01:02:31 +0530 Subject: [PATCH 46/62] [BFCL] Add test group support (#17) Signed-off-by: Tarun Kumar Signed-off-by: Stacy Roberts --- lib/run_bfcl.py | 156 +++++++++++++++++++++----- workloads/deepseek_v4_flash_b200.yaml | 1 + workloads/deepseek_v4_pro_5_h200.yaml | 1 + workloads/glm_5_1_h200.yaml | 1 + workloads/gpt_oss_120b_h200.yaml | 1 + workloads/kimi_k2_5_h200.yaml | 1 + workloads/minimax_m2_5_h200.yaml | 1 + 7 files changed, 134 insertions(+), 28 deletions(-) diff --git a/lib/run_bfcl.py b/lib/run_bfcl.py index 545244e..90a3a82 100644 --- a/lib/run_bfcl.py +++ b/lib/run_bfcl.py @@ -97,39 +97,139 @@ def run_evaluate(model, category): "multi_turn_long_context": "data_multi_turn.csv", } +# BFCL aggregate categories expand to multiple sub-categories at runtime. +# Their scores live in summary columns of the category CSV, not as +# BFCL_v4_{category} columns (and there is no per-aggregate *_score.json). +AGGREGATE_CATEGORY_SCORES = { + "multi_turn": ("data_multi_turn.csv", "Multi Turn Overall Acc"), + "live": ("data_live.csv", "Live Overall Acc"), + "non_live": ("data_non_live.csv", "Non-Live Overall Acc"), + "agentic": ("data_agentic.csv", "Agentic Overall Acc"), + "web_search": ("data_agentic.csv", "Web Search Summary"), + "memory": ("data_agentic.csv", "Memory Summary"), + "all": ("data_overall.csv", "Overall Acc"), + "all_scoring": ("data_overall.csv", "Overall Acc"), +} -def parse_score_from_csv(work_dir: Path, model: str, category: str) -> dict | None: - """Extract per-category accuracy from BFCL V4 aggregate CSV files.""" - csv_name = CATEGORY_TO_CSV.get(category) - csv_candidates = [work_dir / "score" / csv_name] if csv_name else [] - csv_candidates.append(work_dir / "score" / "data_overall.csv") - - bfcl_category = f"BFCL_v4_{category}" - for csv_path in csv_candidates: - if not csv_path.exists(): - continue - with open(csv_path) as f: - reader = csv.DictReader(f) - for row in reader: - model_name = row.get("Model", row.get("model", "")) - if model not in model_name and model.replace("/", "_") not in model_name: - continue - for key, value in row.items(): - if bfcl_category in key: - try: - return {"accuracy": float(value) / 100.0} - except (ValueError, TypeError): - continue - - # Fallback: look for per-category JSONL score files (older BFCL versions) - model_slug = model.replace("/", "_") - for p in (work_dir / "score").rglob(f"*{category}_score.json"): - with open(p) as f: - return json.loads(f.readline()) +def _score_dir(work_dir: Path) -> Path: + return work_dir / "score" + + +def _model_row_match(model: str, row: dict) -> bool: + model_name = row.get("Model", row.get("model", "")) + return model in model_name or model.replace("/", "_") in model_name + + +def _accuracy_from_percentage(value: object) -> float | None: + if value in (None, "", "N/A"): + return None + try: + return float(value) / 100.0 + except (ValueError, TypeError): + return None + + +def _csv_values_for_lookup( + row: dict, *, column: str | None, column_contains: str | None +) -> list[object]: + if column is not None: + return [row.get(column)] + return [value for key, value in row.items() if column_contains in key] + + +def _parse_csv_accuracy( + work_dir: Path, + model: str, + csv_name: str, + *, + column: str | None = None, + column_contains: str | None = None, +) -> dict | None: + """Read a percentage accuracy from a BFCL score CSV for the given model.""" + csv_path = _score_dir(work_dir) / csv_name + if not csv_path.exists(): + return None + + with open(csv_path) as f: + for row in csv.DictReader(f): + if not _model_row_match(model, row): + continue + for value in _csv_values_for_lookup( + row, column=column, column_contains=column_contains + ): + if accuracy := _accuracy_from_percentage(value): + return {"accuracy": accuracy} + return None + + +def _read_score_json(path: Path) -> dict: + with open(path) as f: + return json.loads(f.readline()) + + +def _find_score_json(work_dir: Path, category: str) -> dict | None: + for path in _score_dir(work_dir).rglob(f"*{category}_score.json"): + return _read_score_json(path) + return None + + +def _parse_subcategory_json_average( + work_dir: Path, subcategories: list[str] +) -> dict | None: + accuracies = [] + for subcat in subcategories: + score = _find_score_json(work_dir, subcat) + if score and (acc := score.get("accuracy")) is not None: + accuracies.append(float(acc)) + if not accuracies: + return None + return {"accuracy": sum(accuracies) / len(accuracies)} + + +def _parse_aggregate_score( + work_dir: Path, model: str, category: str +) -> dict | None: + csv_name, column = AGGREGATE_CATEGORY_SCORES[category] + if score := _parse_csv_accuracy(work_dir, model, csv_name, column=column): + return score + + from bfcl_eval.constants.category_mapping import TEST_COLLECTION_MAPPING + + subcategories = TEST_COLLECTION_MAPPING.get(category, []) + if subcategories: + return _parse_subcategory_json_average(work_dir, subcategories) + return None + + +def _parse_leaf_csv_score( + work_dir: Path, model: str, category: str +) -> dict | None: + csv_candidates = [] + if csv_name := CATEGORY_TO_CSV.get(category): + csv_candidates.append(csv_name) + csv_candidates.append("data_overall.csv") + + marker = f"BFCL_v4_{category}" + for csv_name in csv_candidates: + if score := _parse_csv_accuracy( + work_dir, model, csv_name, column_contains=marker + ): + return score return None +def parse_score_from_csv(work_dir: Path, model: str, category: str) -> dict | None: + """Extract per-category accuracy from BFCL V4 score CSVs and JSON files.""" + if category in AGGREGATE_CATEGORY_SCORES: + return _parse_aggregate_score(work_dir, model, category) + + if score := _parse_leaf_csv_score(work_dir, model, category): + return score + + return _find_score_json(work_dir, category) + + def to_lm_eval_format(model: str, category: str, score: dict) -> dict: """Transform BFCL aggregate score into lm_eval-compatible results JSON.""" task_name = f"bfcl_{category}" diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index 694ab37..b0826b8 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -29,6 +29,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 lm_eval: diff --git a/workloads/deepseek_v4_pro_5_h200.yaml b/workloads/deepseek_v4_pro_5_h200.yaml index 27ef33d..a53d751 100644 --- a/workloads/deepseek_v4_pro_5_h200.yaml +++ b/workloads/deepseek_v4_pro_5_h200.yaml @@ -45,6 +45,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/glm_5_1_h200.yaml b/workloads/glm_5_1_h200.yaml index 90bacd8..148cd23 100644 --- a/workloads/glm_5_1_h200.yaml +++ b/workloads/glm_5_1_h200.yaml @@ -36,6 +36,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index 03975f3..31cf6d4 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -33,6 +33,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/kimi_k2_5_h200.yaml b/workloads/kimi_k2_5_h200.yaml index 684bcca..0f6ec46 100644 --- a/workloads/kimi_k2_5_h200.yaml +++ b/workloads/kimi_k2_5_h200.yaml @@ -33,6 +33,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: diff --git a/workloads/minimax_m2_5_h200.yaml b/workloads/minimax_m2_5_h200.yaml index 58f7349..c5dd892 100644 --- a/workloads/minimax_m2_5_h200.yaml +++ b/workloads/minimax_m2_5_h200.yaml @@ -36,6 +36,7 @@ bfcl: - multiple - parallel - parallel_multiple + - multi_turn num_threads: 8 vllm_bench: From 8181798efe8ef96227f7a597386f267065132d42 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 11 Jun 2026 16:53:01 -0700 Subject: [PATCH 47/62] Switch vllm_bench from speed_bench to random + ignore_eos (#20) Co-authored-by: Claude Opus 4.6 (1M context) Signed-off-by: Stacy Roberts --- README.md | 4 +++- lib/run_vllm_bench.sh | 4 +++- lib/server.sh | 3 +++ workloads/deepseek_v4_pro_5_h200.yaml | 6 ++---- workloads/glm_5_1_h200.yaml | 6 ++---- workloads/gpt_oss_120b_h200.yaml | 6 ++---- workloads/kimi_k2_5_h200.yaml | 6 ++---- workloads/minimax_m2_5_h200.yaml | 6 ++---- workloads/nemotron_3_super_5_h200.yaml | 6 ++---- workloads/qwen3_5_h200.yaml | 6 ++---- 10 files changed, 23 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 166d691..490e3f5 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,8 @@ bfcl: # function-calling eval (optional) vllm_bench: # perf runs (optional) — fed to the perf dashboard configs: - name: 1k-in-1k-out-conc-256 - dataset: random # or speed_bench + backend: openai # /v1/completions — exact ISL/OSL, no chat template + dataset: random # synthetic fixed-length throughput dataset input_len: 1024 output_len: 1024 num_prompts: 500 @@ -90,6 +91,7 @@ A few things worth knowing: - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass. +- **`vllm_bench` uses the `random` dataset with `--ignore-eos`** so every request prefills exactly `input_len` and decodes exactly `output_len` tokens — that's what makes the per-GPU decode throughput meaningful. Pair it with `backend: openai` (the `/v1/completions` endpoint) for exact token control. Avoid `dataset: speed_bench` for throughput numbers: it requires `--skip-tokenizer-init`, which makes `vllm bench serve` cap every request at a single output token, so output throughput reads as ~0. - **`bfcl` may need tool-call serve args.** Some models require `--enable-auto-tool-choice` and `--tool-call-parser` for function-calling; the parser warns if `--tool-call-parser` is absent. Each category runs as a separate generate + evaluate pass; scores appear on the eval dashboard as `bfcl_` tasks. For everything else (the full set of supported fields, defaults, validation rules), the existing files in `workloads/` are the working reference and `lib/parse_workload.py` is the source of truth. diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index 3a4fa77..e8a6240 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -110,7 +110,9 @@ run_vllm_bench() { case "$dataset" in random) - cmd+=(--random-input-len "$input_len" --random-output-len "$output_len") + # --ignore-eos forces every request to emit the full output_len; without it + # the model can stop early on the random prompt and decode throughput collapses. + cmd+=(--random-input-len "$input_len" --random-output-len "$output_len" --ignore-eos) ;; speed_bench) [[ -z "$speed_bench_dataset_subset" ]] && speed_bench_dataset_subset="qualitative" diff --git a/lib/server.sh b/lib/server.sh index 7ee6345..b500882 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -66,6 +66,9 @@ start_server() { # "$model" --port "$port" $serve_args "$model" --port "$port" "${serve_args_arr[@]}" + # Install pytest to avoid cupy.testing import failure during torch.compile + docker exec "$container" pip install -q pytest 2>/dev/null || true + echo "--- :memo: streaming vllm logs" ( docker logs -f "$container" 2>&1 | stdbuf -oL -eL sed 's/^/[vllm] /' ) & VLLM_LOGS_PID=$! diff --git a/workloads/deepseek_v4_pro_5_h200.yaml b/workloads/deepseek_v4_pro_5_h200.yaml index a53d751..18e7b38 100644 --- a/workloads/deepseek_v4_pro_5_h200.yaml +++ b/workloads/deepseek_v4_pro_5_h200.yaml @@ -51,11 +51,9 @@ bfcl: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy diff --git a/workloads/glm_5_1_h200.yaml b/workloads/glm_5_1_h200.yaml index 148cd23..002dc31 100644 --- a/workloads/glm_5_1_h200.yaml +++ b/workloads/glm_5_1_h200.yaml @@ -42,11 +42,9 @@ bfcl: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index 31cf6d4..b16202b 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -39,11 +39,9 @@ bfcl: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy diff --git a/workloads/kimi_k2_5_h200.yaml b/workloads/kimi_k2_5_h200.yaml index 0f6ec46..8556129 100644 --- a/workloads/kimi_k2_5_h200.yaml +++ b/workloads/kimi_k2_5_h200.yaml @@ -39,11 +39,9 @@ bfcl: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy diff --git a/workloads/minimax_m2_5_h200.yaml b/workloads/minimax_m2_5_h200.yaml index c5dd892..8d796b4 100644 --- a/workloads/minimax_m2_5_h200.yaml +++ b/workloads/minimax_m2_5_h200.yaml @@ -42,11 +42,9 @@ bfcl: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy diff --git a/workloads/nemotron_3_super_5_h200.yaml b/workloads/nemotron_3_super_5_h200.yaml index 9dc055d..ff76e83 100644 --- a/workloads/nemotron_3_super_5_h200.yaml +++ b/workloads/nemotron_3_super_5_h200.yaml @@ -32,11 +32,9 @@ lm_eval: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy diff --git a/workloads/qwen3_5_h200.yaml b/workloads/qwen3_5_h200.yaml index 6ed2054..d0ec727 100644 --- a/workloads/qwen3_5_h200.yaml +++ b/workloads/qwen3_5_h200.yaml @@ -30,11 +30,9 @@ lm_eval: vllm_bench: configs: - name: 8k-in-1k-out-conc-128 - backend: openai-chat - dataset: speed_bench + backend: openai + dataset: random input_len: 8192 output_len: 1024 num_prompts: 512 max_concurrency: 128 - speed_bench_dataset_subset: throughput_8k - speed_bench_category: low_entropy From 5e229a6493bcbe7cffafd52d59e225f20462a6ab Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Sat, 13 Jun 2026 03:18:59 +0530 Subject: [PATCH 48/62] [BFCL] Remove parallel tool calling test from gpt-oss model (#21) Signed-off-by: Tarun Kumar Signed-off-by: Stacy Roberts --- workloads/gpt_oss_120b_h200.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/workloads/gpt_oss_120b_h200.yaml b/workloads/gpt_oss_120b_h200.yaml index b16202b..162b10f 100644 --- a/workloads/gpt_oss_120b_h200.yaml +++ b/workloads/gpt_oss_120b_h200.yaml @@ -31,8 +31,6 @@ bfcl: test_categories: - simple_python - multiple - - parallel - - parallel_multiple - multi_turn num_threads: 8 From baf37905d19a67c5d251787a848958f0808cd380 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Thu, 25 Jun 2026 09:23:33 -0500 Subject: [PATCH 49/62] Fixed errant call in parse_workload and adding moe sweep Signed-off-by: sroberts-amd --- .gitignore | 1 - lib/parse_workload.py | 2 +- .../moe_sweep_deepseek_r1_0528_mi355x.yaml | 61 +++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml diff --git a/.gitignore b/.gitignore index 80311f4..3913b14 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ diff --git a/lib/parse_workload.py b/lib/parse_workload.py index 036fe75..0662be8 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -284,7 +284,7 @@ def validate_attention_backends(backends: list, path: str) -> None: "some models may need it for function-calling", file=sys.stderr, ) - _validate_bfcl_limits(bfcl, path) + def max_test_cases_for_category(bfcl: dict, category: str) -> int | None: diff --git a/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..cd2353f --- /dev/null +++ b/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,61 @@ +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: moe-sweep-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --moe-backend AITER + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + - default + #- ROCM_AITER_MLA #default 2026-06-18 + #- ROCM_AITER_TRITON_MLA + #- ROCM_AITER_MLA_SPARSE #not supported Reason: ['non-sparse not supported'] + +vllm_bench: + configs: + # - name: isl1024-osl1024-conc4 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 \ No newline at end of file From d60e1079d734352221af518a6683233b84e92bb5 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Thu, 25 Jun 2026 17:19:50 -0500 Subject: [PATCH 50/62] Moved attention sweep logic out of main run.sh file updated gen_report.py to reflect movement of results files. Signed-off-by: sroberts-amd --- gen_report.py | 15 +- lib/run.sh | 188 ++++++------------ lib/run_attn_sweep.sh | 155 +++++++++++++++ .../attn_sweep_amd_gpt_oss_120b_mi355x.yaml | 2 +- 4 files changed, 235 insertions(+), 125 deletions(-) create mode 100755 lib/run_attn_sweep.sh diff --git a/gen_report.py b/gen_report.py index 0e739b4..c8a6da0 100644 --- a/gen_report.py +++ b/gen_report.py @@ -79,10 +79,23 @@ def ingest(txt_path: Path, backend_override: str | None = None) -> str | None: if name and default_backend is None: default_backend = name - # attn-BACKEND subdirs — non-default backends + # attn-BACKEND subdirs — explicit backend sweep results for sub in sorted(model_dir.iterdir()): if sub.is_dir() and sub.name.startswith("attn-"): backend_name = sub.name[len("attn-"):] + if backend_name == "default": + attn_file = sub / "attn_backend.txt" + if attn_file.exists(): + raw = attn_file.read_text() + m = OVERRIDE_PATTERN.search(raw) + if m: + backend_name = m.group(1) + else: + # Fall back to the last non-empty word on the last line + # (covers "Using XYZ backend" style lines). + last = raw.strip().splitlines()[-1].strip() + backend_name = last.split()[0] if last else "default" + # backend_name may still be "default" if attn_backend.txt is absent. for txt in sorted(sub.glob("*-summary.txt")): ingest(txt, backend_override=backend_name) diff --git a/lib/run.sh b/lib/run.sh index 8ed061e..29cd479 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -2,11 +2,7 @@ # Orchestrate a workload: bring up vLLM, then dispatch each task to the # helper script for its type. # -# When vllm.attention_backends is set in the workload YAML, the server is -# started once per backend (with --attention-backend appended to serve_args), -# and the full eval suite runs for each. Results land in -# results//attn-/ instead of results//. - +# Usage: ./lib/run.sh workloads/qwen3_5_h200.yaml set -euo pipefail WORKLOAD="${1:?usage: $0 }" @@ -31,124 +27,70 @@ if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$ BENCH_TRUST_REMOTE_CODE=true fi -# Build the list of attention backends to sweep. An empty -# WORKLOAD_ATTENTION_BACKENDS means "run once with whatever vLLM picks" and -# we represent that as a single sentinel entry so the loop always executes. -mapfile -t ATTN_BACKENDS <<< "${WORKLOAD_ATTENTION_BACKENDS}" -if [[ "${#ATTN_BACKENDS[@]}" -eq 0 || ( "${#ATTN_BACKENDS[@]}" -eq 1 && -z "${ATTN_BACKENDS[0]}" ) ]]; then - ATTN_BACKENDS=("default") +# When an attention-backend sweep is configured, delegate entirely to the +# sweep script and exit. Everything below is the single-backend path. +if [[ -n "${WORKLOAD_ATTENTION_BACKENDS:-}" ]]; then + exec "$DIR/run_attn_sweep.sh" "$WORKLOAD" fi -for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do - if [[ "$ATTN_BACKEND" == "default" ]]; then - echo "=== :brain: attention backend: (vLLM default)" - RESULTS_DIR="results/${WORKLOAD_NAME}" - EFFECTIVE_SERVE_ARGS="$WORKLOAD_SERVE_ARGS" - else - echo "=== :brain: attention backend: ${ATTN_BACKEND}" - RESULTS_DIR="results/${WORKLOAD_NAME}/attn-${ATTN_BACKEND}" - # --attention-backend is a vLLM server arg, not an env var. - EFFECTIVE_SERVE_ARGS="${WORKLOAD_SERVE_ARGS} --attention-backend ${ATTN_BACKEND}" - fi - mkdir -p "$RESULTS_DIR" - - CONTAINER="perf-eval-${WORKLOAD_NAME}-${ATTN_BACKEND}-$$" - - trap 'stop_server "$CONTAINER"' EXIT - - if ! start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ - "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME"; then - echo "^^^ +++ ERROR: start_server failed for backend ${ATTN_BACKEND}; skipping" >&2 - stop_server "$CONTAINER" - trap - EXIT - drain_gpu - continue - fi - - if ! wait_healthy "$PORT"; then - echo "^^^ +++ ERROR: vLLM never became healthy for backend ${ATTN_BACKEND}; skipping" >&2 - stop_server "$CONTAINER" - trap - EXIT - drain_gpu - continue - fi - - if [[ "$ATTN_BACKEND" == "default" ]]; then - echo "--- :mag: attention backend selected by vLLM:" - _backend_lines="" - if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then - _backend_lines=$(grep -E "(Overriding with|Using [A-Z_]+ backend)" \ - "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null) || true - else - _backend_lines=$(docker logs "$CONTAINER" 2>&1 \ - | grep -E "(Overriding with|Using [A-Z_]+ backend)") || true - fi - if [[ -n "$_backend_lines" ]]; then - echo "$_backend_lines" | sed 's/^/ /' - else - echo " (backend selection lines not found in log)" - fi - fi - - # vllm bench serve runs first so we can validate perf flow without waiting - # on a full lm_eval pass. Each config's raw json lands in - # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the - # perf dashboard ingest endpoint. - while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do - [[ -z "$bname" ]] && continue - if ! run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ - "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ - "$conc" "$speed_subset" "$speed_category" \ - "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR"; then - echo "^^^ +++ ERROR: run_vllm_bench failed for ${bname} (backend ${ATTN_BACKEND}); skipping run" >&2 - continue - fi - - python3 "$DIR/ingest_perf.py" \ - --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ - --device "$WORKLOAD_BENCH_DEVICE" \ - --tp "$WORKLOAD_BENCH_TP" \ - --precision "$WORKLOAD_BENCH_PRECISION" \ - --model "$WORKLOAD_MODEL" \ - --image "$WORKLOAD_IMAGE" \ - --isl "$isl" --osl "$osl" --conc "$conc" || true - done <<< "$WORKLOAD_VLLM_BENCH_TSV" - - if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then - echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" - stop_server "$CONTAINER" - trap - EXIT - drain_gpu - continue - fi - - while IFS=$'\t' read -r task fewshot model_args; do - [[ -z "$task" ]] && continue - run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ - "$model_args" "$RESULTS_DIR" - - python3 "$DIR/ingest.py" \ - --results-dir "${RESULTS_DIR}/${task}" \ - --workload "$WORKLOAD_NAME" \ - --task "$task" \ - ${INGEST_NO_SAMPLES:+--no-samples} || true - done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" - - # bfcl function-calling eval - while IFS=$'\t' read -r category num_threads temperature; do - [[ -z "$category" ]] && continue - echo "--- :phone: bfcl ${category}" - python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ - "$category" "$num_threads" "$temperature" "$RESULTS_DIR" - - python3 "$DIR/ingest.py" \ - --results-dir "${RESULTS_DIR}/bfcl-${category}" \ - --workload "$WORKLOAD_NAME" \ - --task "bfcl_${category}" \ - --no-samples || true - done <<< "$WORKLOAD_BFCL_TSV" +CONTAINER="perf-eval-${WORKLOAD_NAME}-$$" +RESULTS_DIR="results/${WORKLOAD_NAME}" +mkdir -p "$RESULTS_DIR" + +trap 'stop_server "$CONTAINER"' EXIT + +start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$WORKLOAD_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" +wait_healthy "$PORT" + +# vllm bench serve runs first so we can validate perf flow without waiting +# on a full lm_eval pass. Each config's raw json lands in +# $RESULTS_DIR/bench-.json and is then transformed and POSTed to the +# perf dashboard ingest endpoint. +while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do + [[ -z "$bname" ]] && continue + run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR" + + python3 "$DIR/ingest_perf.py" \ + --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ + --device "$WORKLOAD_BENCH_DEVICE" \ + --tp "$WORKLOAD_BENCH_TP" \ + --precision "$WORKLOAD_BENCH_PRECISION" \ + --model "$WORKLOAD_MODEL" \ + --image "$WORKLOAD_IMAGE" \ + --isl "$isl" --osl "$osl" --conc "$conc" || true +done <<< "$WORKLOAD_VLLM_BENCH_TSV" + +if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" + exit 0 +fi - stop_server "$CONTAINER" - trap - EXIT - drain_gpu -done +while IFS=$'\t' read -r task fewshot model_args; do + [[ -z "$task" ]] && continue + run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ + "$model_args" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/${task}" \ + --workload "$WORKLOAD_NAME" \ + --task "$task" \ + ${INGEST_NO_SAMPLES:+--no-samples} || true +done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + +# bfcl function-calling eval +while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true +done <<< "$WORKLOAD_BFCL_TSV" diff --git a/lib/run_attn_sweep.sh b/lib/run_attn_sweep.sh new file mode 100755 index 0000000..8bd11b7 --- /dev/null +++ b/lib/run_attn_sweep.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Attention-backend sweep: run the full eval suite once per backend listed in +# vllm.attention_backends in the workload YAML. Results land in +# results//attn-/ for each backend. +# +# Invoked by run.sh via exec when WORKLOAD_ATTENTION_BACKENDS is non-empty. +# Can also be run directly: ./lib/run_attn_sweep.sh workloads/foo.yaml +# +# Usage: ./lib/run_attn_sweep.sh + +set -euo pipefail + +WORKLOAD="${1:?usage: $0 }" +[[ -f "$WORKLOAD" ]] || { echo "not found: $WORKLOAD" >&2; exit 2; } + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$DIR/server.sh" +# shellcheck disable=SC1091 +source "$DIR/run_lm_eval.sh" +# shellcheck disable=SC1091 +source "$DIR/run_vllm_bench.sh" +WORKLOAD_EXPORTS="$(python3 "$DIR/parse_workload.py" "$WORKLOAD")" +eval "$WORKLOAD_EXPORTS" +export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME + +PORT=8000 +BASE_URL="http://localhost:${PORT}" +BENCH_TRUST_REMOTE_CODE=false +if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || + [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then + BENCH_TRUST_REMOTE_CODE=true +fi + +mapfile -t ATTN_BACKENDS <<< "$WORKLOAD_ATTENTION_BACKENDS" + +for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do + [[ -z "$ATTN_BACKEND" ]] && continue + + if [[ "$ATTN_BACKEND" == "default" ]]; then + echo "=== :brain: attention backend: (vLLM default)" + RESULTS_DIR="results/${WORKLOAD_NAME}/attn-default" + EFFECTIVE_SERVE_ARGS="$WORKLOAD_SERVE_ARGS" + else + echo "=== :brain: attention backend: ${ATTN_BACKEND}" + RESULTS_DIR="results/${WORKLOAD_NAME}/attn-${ATTN_BACKEND}" + # --attention-backend is a vLLM server arg, not an env var. + EFFECTIVE_SERVE_ARGS="${WORKLOAD_SERVE_ARGS} --attention-backend ${ATTN_BACKEND}" + fi + mkdir -p "$RESULTS_DIR" + + CONTAINER="perf-eval-${WORKLOAD_NAME}-${ATTN_BACKEND}-$$" + + trap 'stop_server "$CONTAINER"' EXIT + + if ! start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME"; then + echo "^^^ +++ ERROR: start_server failed for backend ${ATTN_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if ! wait_healthy "$PORT"; then + echo "^^^ +++ ERROR: vLLM never became healthy for backend ${ATTN_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if [[ "$ATTN_BACKEND" == "default" ]]; then + echo "--- :mag: attention backend selected by vLLM:" + _backend_lines="" + if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then + _backend_lines=$(grep -E "(Overriding with|Using [A-Z_]+ backend)" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null) || true + else + _backend_lines=$(docker logs "$CONTAINER" 2>&1 \ + | grep -E "(Overriding with|Using [A-Z_]+ backend)") || true + fi + if [[ -n "$_backend_lines" ]]; then + echo "$_backend_lines" | sed 's/^/ /' + echo "$_backend_lines" > "${RESULTS_DIR}/attn_backend.txt" + else + echo " (backend selection lines not found in log)" + echo "unknown" > "${RESULTS_DIR}/attn_backend.txt" + fi + else + echo "$ATTN_BACKEND" > "${RESULTS_DIR}/attn_backend.txt" + fi + + # vllm bench serve runs first so we can validate perf flow without waiting + # on a full lm_eval pass. Each config's raw json lands in + # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the + # perf dashboard ingest endpoint. + while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do + [[ -z "$bname" ]] && continue + if ! run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR"; then + echo "^^^ +++ ERROR: run_vllm_bench failed for ${bname} (backend ${ATTN_BACKEND}); skipping run" >&2 + continue + fi + + python3 "$DIR/ingest_perf.py" \ + --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ + --device "$WORKLOAD_BENCH_DEVICE" \ + --tp "$WORKLOAD_BENCH_TP" \ + --precision "$WORKLOAD_BENCH_PRECISION" \ + --model "$WORKLOAD_MODEL" \ + --image "$WORKLOAD_IMAGE" \ + --isl "$isl" --osl "$osl" --conc "$conc" || true + done <<< "$WORKLOAD_VLLM_BENCH_TSV" + + if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + while IFS=$'\t' read -r task fewshot model_args; do + [[ -z "$task" ]] && continue + run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ + "$model_args" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/${task}" \ + --workload "$WORKLOAD_NAME" \ + --task "$task" \ + ${INGEST_NO_SAMPLES:+--no-samples} || true + done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + + # bfcl function-calling eval + while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true + done <<< "$WORKLOAD_BFCL_TSV" + + stop_server "$CONTAINER" + trap - EXIT + drain_gpu +done diff --git a/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml index dd46b4b..bc784aa 100644 --- a/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml +++ b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -16,7 +16,7 @@ vllm: attention_backends: - default - TRITON_ATTN - #- ROCM_AITER_FA default 2026-06-18 + #- ROCM_AITER_FA # incompatible in v0.22.0 was default 2026-06-18 #- ROCM_ATTN #fails for attention sinks not supported - ROCM_AITER_UNIFIED_ATTN From 141fadfe9660eef90dda838e7edd0310122f4931 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Fri, 26 Jun 2026 15:55:52 -0500 Subject: [PATCH 51/62] adding in moe-backend sweep. Still issue with report generation Signed-off-by: sroberts-amd --- README.md | 9 +- gen_report.py | 220 ++++++++++++++++++ lib/parse_workload.py | 40 +++- lib/run.sh | 5 + lib/run_moe_sweep.sh | 160 +++++++++++++ lib/run_vllm_bench.sh | 7 +- .../moe_sweep_amd_gpt_oss_120b_mi355x.yaml | 61 +++++ .../moe_sweep_deepseek_r1_0528_mi355x.yaml | 22 +- 8 files changed, 508 insertions(+), 16 deletions(-) create mode 100755 lib/run_moe_sweep.sh create mode 100644 workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml diff --git a/README.md b/README.md index a90b410..dc4ecbd 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ CLAUDE.md agent conventions and detailed Buildkite workflow A recipe has top-level metadata plus up to three eval blocks: -- **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides, optional `attention_backends` list). Required. +- **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides, optional `attention_backends` or `moe_backends` list). Required. - **`lm_eval:`** — *what accuracy to measure.* Lists lm-evaluation-harness tasks to run against the live server (e.g. `gsm8k`, `aime25`). Each task's score is saved under `results///`. Optional. - **`vllm_bench:`** — *what perf to measure.* Lists `vllm bench serve` configs (input/output lengths, concurrency, dataset). Raw JSON is saved and ingested into the perf dashboard. Optional. - **`bfcl:`** — *function-calling eval.* Runs [BFCL](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) test categories against the live server. Some models need `--enable-auto-tool-choice` and `--tool-call-parser` in `serve_args`. Results are transformed to lm_eval format and ingested as `bfcl_` tasks. Optional. @@ -51,6 +51,10 @@ vllm: # how the server is brought up attention_backends: # optional; list of VLLM_ATTENTION_BACKEND values - FLASH_ATTN # when set, the full eval suite runs once per - FLASHINFER # backend; results land in attn-/ subdirs + # moe_backends: # optional; list of --moe-backend values + # - default # "default" means no --moe-backend flag (vLLM picks) + # - AITER # results land in moe-/ subdirs + # attention_backends and moe_backends are mutually exclusive in a single workload lm_eval: # accuracy tasks (optional) model_args: # workload-level defaults, merged into every task @@ -90,6 +94,7 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: - **`vllm.attention_backends`** is an optional list of vLLM attention backend names (`FLASH_ATTN`, `FLASHINFER`, `XFORMERS`, `TRITON_ATTN`, `TRITON_MLA`, `ROCM_FLASH`, `PAGED_ATTENTION`,`ROCM_AITER_FA`,`ROCM_AITER_UNIFIED_ATTN`, `ROCM_ATTN`,`ROCM_AITER_MLA`,`ROCM_AITER_MLA_SPARSE`, `ROCM_AITER_TRITON_MLA`). When set, the orchestrator starts the server once per backend — adding `--attention-backend $ATTN_BACKEND` — and runs the complete eval suite (bench, lm_eval, bfcl) for each. Results are stored under `results//attn-/` so every backend gets its own isolated output directory. Without this field, the server starts once with whatever attention backend vLLM selects by default and results go to `results//` as usual. See `workloads/attn-sweep-gpt-oss-120b-mi355x.yaml` for an example. +- **`vllm.moe_backends`** is an optional list of vLLM `--moe-backend` values (`default`, `AITER`, `TRITON`, `FUSED_MOE`, `deep_gemm_mega_moe`). When set, the orchestrator starts the server once per backend — adding `--moe-backend $MOE_BACKEND` for non-default values — and runs the complete eval suite for each. Results are stored under `results//moe-/`. The special value `"default"` means no `--moe-backend` flag (vLLM picks the backend automatically). **Do not include `--moe-backend` in `serve_args` when using this field** — the sweep script owns that flag. `attention_backends` and `moe_backends` are mutually exclusive in a single workload. See `workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml` for an example. - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. @@ -157,6 +162,8 @@ If reports from previous rounds are already present in the directory, `benchmark Each per-model report shows attention backend results side by side, with tabs for each input sequence length, color-coded best/worst values per metric, and percentage deltas relative to the default backend. +**MoE backend sweep reports** are generated automatically alongside the attention sweep reports. For each model directory that contains `moe-*` result subdirectories, `gen_report.py` writes a `moe-benchmark-.html` file. A `moe-benchmark-index.html` landing page is also produced, covering all models with MoE sweep data. Open it in a browser to tab between models — same layout and features as the attention backend report. + ## Agents `CLAUDE.md` has conventions for AI agents working in this repo: smoke-testing changes, launching Buildkite builds for a chosen branch/commit, and the AI-assistance disclosure rule for PRs and commits. diff --git a/gen_report.py b/gen_report.py index c8a6da0..1488c63 100644 --- a/gen_report.py +++ b/gen_report.py @@ -34,6 +34,10 @@ r"attention_backend:\s*(\S+).*?isl:\s*(\d+)\s+osl:\s*(\d+)\s+conc:\s*(\d+)", re.DOTALL, ) +MOE_HEADER_PATTERN = re.compile( + r"moe_backend:\s*(\S+).*?isl:\s*(\d+)\s+osl:\s*(\d+)\s+conc:\s*(\d+)", + re.DOTALL, +) OVERRIDE_PATTERN = re.compile(r"Overriding with (\S+) out of potential backends") @@ -53,6 +57,19 @@ def parse_txt(path: Path) -> dict | None: return {"backend": backend, "isl": isl, "osl": osl, "conc": conc, "metrics": metrics} +def parse_txt_moe(path: Path) -> dict | None: + text = path.read_text(errors="replace") + m = MOE_HEADER_PATTERN.search(text) + if not m: + return None + backend, isl, osl, conc = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4)) + metrics = {} + for name, pat in METRIC_PATTERNS.items(): + hit = re.search(pat, text) + metrics[name] = float(hit.group(1)) if hit else None + return {"backend": backend, "isl": isl, "osl": osl, "conc": conc, "metrics": metrics} + + # ── Directory scan ──────────────────────────────────────────────────────────── def collect_model_dir(model_dir: Path) -> tuple[dict, str | None]: @@ -102,6 +119,46 @@ def ingest(txt_path: Path, backend_override: str | None = None) -> str | None: return dict(data), default_backend +def collect_model_dir_moe(model_dir: Path) -> tuple[dict, str | None]: + """Return ({backend -> {(isl,osl,conc) -> metrics_dict}}, default_backend_name) + + Looks for moe-* subdirectories instead of attn-* subdirectories. + """ + data = defaultdict(dict) + default_backend: str | None = None + + def ingest(txt_path: Path, backend_override: str | None = None) -> str | None: + rec = parse_txt_moe(txt_path) + if rec is None: + return None + backend = backend_override or rec["backend"] + key = (rec["isl"], rec["osl"], rec["conc"]) + data[backend][key] = rec["metrics"] + return backend + + # moe-BACKEND subdirs — explicit moe backend sweep results + for sub in sorted(model_dir.iterdir()): + if sub.is_dir() and sub.name.startswith("moe-"): + backend_name = sub.name[len("moe-"):] + if backend_name == "default": + moe_file = sub / "moe_backend.txt" + if moe_file.exists(): + raw = moe_file.read_text() + # Look for vLLM's moe-specific log line, e.g.: + # Using AITER Fp8 MoE backend out of potential backends: [...] + m = re.search(r"Using (.+?) MoE backend", raw) + if m: + backend_name = m.group(1) + + # backend_name stays "default" if the moe backend line is absent. + for txt in sorted(sub.glob("*-summary.txt")): + name = ingest(txt, backend_override=backend_name) + if name and default_backend is None: + default_backend = name + + return dict(data), default_backend + + # ── HTML generation ─────────────────────────────────────────────────────────── CSS = """ @@ -566,6 +623,123 @@ def build_index_html(reports: list[tuple[str, str]]) -> str: """ +def build_html_moe(model_name: str, backend_data: dict, default_backend: str | None) -> str: + """Build a per-model MoE backend benchmark HTML page.""" + all_backends = sorted(backend_data.keys()) + if default_backend and default_backend in all_backends: + all_backends.remove(default_backend) + all_backends = [default_backend] + all_backends + + all_keys: set[tuple] = set() + for bdata in backend_data.values(): + all_keys.update(bdata.keys()) + + data_dict = {} + for (isl, osl, conc) in sorted(all_keys): + dk = f"isl{isl}-conc{conc}" + row: dict[str, list] = {} + for mname in METRIC_PATTERNS: + vals = [] + for b in all_backends: + bdata = backend_data.get(b, {}) + point = bdata.get((isl, osl, conc), {}) + vals.append(point.get(mname) if point else None) + row[mname] = vals + data_dict[dk] = row + + isl_values = sorted({k[0] for k in all_keys}) + osl_values = sorted({k[1] for k in all_keys}) + osl_label = "/".join(str(o) for o in osl_values) + + backends_js = json.dumps([ + {"key": b.lower().replace("_", ""), "label": b, "isDefault": i == 0} + for i, b in enumerate(all_backends) + ], indent=2) + + data_js = json.dumps(data_dict, indent=2) + + tabs_html = "" + tab_divs = "" + for i, isl in enumerate(isl_values): + tab_id = f"isl{isl}" + active = " active" if i == 0 else "" + tabs_html += f'
ISL {isl} / OSL {osl_label}
\n' + tab_divs += f'
\n' + + subtitle = f"Model: {model_name}" + + html = f""" + + + + + MoE Benchmark — {model_name} + + + + +

MoE Backend Benchmark

+

{subtitle}

+ +
+
Best in row
+
Worst in row
+ · deltas relative to default (first) MoE backend +
+ +
+{tabs_html}
+ +{tab_divs} + + + +""" + return html + + +def build_index_html_moe(reports: list[tuple[str, str]]) -> str: + """Build a single-page wrapper that tabs between per-model MoE report iframes.""" + tabs_html = "" + frames_html = "" + for i, (label, filename) in enumerate(reports): + frame_id = f"frame-{i}" + tabs_html += f'
{label}
\n' + frames_html += f' \n' + + return f""" + + + + + MoE Backend Benchmark Reports + + + + +
+

MoE Backend Benchmarks

+
+{tabs_html}
+
+ +
+{frames_html}
+ + + + +""" + + # ── Main ────────────────────────────────────────────────────────────────────── def model_name_from_dir(dir_name: str) -> str: @@ -574,6 +748,12 @@ def model_name_from_dir(dir_name: str) -> str: return name +def model_name_from_dir_moe(dir_name: str) -> str: + """Best-effort human label from directory name like moe_sweep_deepseek_r1_0528_mi355x.""" + name = re.sub(r"^moe_sweep_", "", dir_name) + return name + + def main(): if not RESULTS_DIR.exists(): sys.exit(f"results/ not found at {RESULTS_DIR}") @@ -614,6 +794,46 @@ def main(): index_path.write_text(build_index_html(reports)) print(f" wrote benchmark-index.html ({len(reports)} model(s))") + # ── MoE backend sweep reports ───────────────────────────────────────────── + moe_reports: list[tuple[str, str]] = [] + + for model_dir in model_dirs: + # Only process directories that contain at least one moe-* subdir + has_moe = any( + sub.is_dir() and sub.name.startswith("moe-") + for sub in model_dir.iterdir() + ) + if not has_moe: + continue + + backend_data, default_backend = collect_model_dir_moe(model_dir) + if not backend_data: + print(f" skip {model_dir.name} (moe) — no parseable summary files") + continue + + model_name = model_name_from_dir_moe(model_dir.name) + html = build_html_moe(model_name, backend_data, default_backend) + + out_filename = f"moe-benchmark-{model_dir.name}.html" + out_path = OUT_DIR / out_filename + out_path.write_text(html) + backends = list(backend_data.keys()) + points = sum(len(v) for v in backend_data.values()) + print(f" wrote {out_filename} ({len(backends)} backends, {points} test points)") + moe_reports.append((model_name, out_filename)) + + # Pick up any pre-existing moe-benchmark-*.html files not produced this run. + existing_moe = {fname for _, fname in moe_reports} + for html_path in sorted(OUT_DIR.glob("moe-benchmark-*.html")): + if html_path.name not in existing_moe and html_path.name != "moe-benchmark-index.html": + label = model_name_from_dir_moe(html_path.stem.removeprefix("moe-benchmark-")) + moe_reports.append((label, html_path.name)) + + if moe_reports: + index_path = OUT_DIR / "moe-benchmark-index.html" + index_path.write_text(build_index_html_moe(moe_reports)) + print(f" wrote moe-benchmark-index.html ({len(moe_reports)} model(s))") + if __name__ == "__main__": main() diff --git a/lib/parse_workload.py b/lib/parse_workload.py index 0662be8..94e783c 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -21,7 +21,7 @@ TASK_FIELDS = {"name", "num_fewshot", "model_args"} VLLM_FIELDS = { - "model", "image", "serve_args", "env", "attention_backends", + "model", "image", "serve_args", "env", "attention_backends", "moe_backends", } KNOWN_ATTENTION_BACKENDS = { "default", @@ -30,6 +30,13 @@ "ROCM_AITER_UNIFIED_ATTN", "ROCM_ATTN", "ROCM_AITER_MLA", "ROCM_AITER_MLA_SPARSE", "ROCM_AITER_TRITON_MLA" } +KNOWN_MOE_BACKENDS = { + "default", + "AITER", "TRITON", "FUSED_MOE", + "AITER_MXFP4_BF16", "AITER_MXFP4_FP8", "TRITON_UNFUSED", + "AITER_MXFP4_MXFP4" +} + BENCH_FIELDS = { "name", "backend", "dataset", "input_len", "output_len", "num_prompts", "max_concurrency", @@ -287,6 +294,19 @@ def validate_attention_backends(backends: list, path: str) -> None: +def validate_moe_backends(backends: list, path: str) -> None: + if not backends: + sys.exit(f"{path}: vllm.moe_backends must not be empty if specified") + for b in backends: + if b not in KNOWN_MOE_BACKENDS: + sys.exit( + f"{path}: unknown moe backend {b!r}; " + f"known: {', '.join(sorted(KNOWN_MOE_BACKENDS))}" + ) + if len(backends) != len(set(backends)): + sys.exit(f"{path}: duplicate entries in vllm.moe_backends") + + def max_test_cases_for_category(bfcl: dict, category: str) -> int | None: cases = bfcl.get("max_test_cases") if isinstance(cases, int): @@ -352,6 +372,21 @@ def main(path: str) -> None: if attention_backends: validate_attention_backends(attention_backends, path) + moe_backends = vllm.get("moe_backends") or [] + if moe_backends: + validate_moe_backends(moe_backends, path) + if re.search(r"(^|[\s])--moe-backend(\s|=)", serve_args): + sys.exit( + f"{path}: vllm.moe_backends and --moe-backend in serve_args are mutually exclusive; " + "remove --moe-backend from serve_args when using the moe_backends sweep" + ) + + if attention_backends and moe_backends: + sys.exit( + f"{path}: vllm.attention_backends and vllm.moe_backends are mutually exclusive; " + "run attention and moe sweeps in separate workloads" + ) + image, vllm_commit = resolve_image(vllm, profile) env = {**(profile.get("env") or {}), **(vllm.get("env") or {})} if "HF_HOME" not in env and profile.get("hf_home"): @@ -376,8 +411,9 @@ def main(path: str) -> None: emit("BENCH_TP", tp) #emit("BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or "")) # One backend per line; empty string when not specified (run.sh treats this - # as a single "default" pass with no VLLM_ATTENTION_BACKEND override). + # as a single "default" pass with no override). emit("ATTENTION_BACKENDS", "\n".join(attention_backends)) + emit("MOE_BACKENDS", "\n".join(moe_backends)) emit( "BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or ""), diff --git a/lib/run.sh b/lib/run.sh index 29cd479..a12bb23 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -33,6 +33,11 @@ if [[ -n "${WORKLOAD_ATTENTION_BACKENDS:-}" ]]; then exec "$DIR/run_attn_sweep.sh" "$WORKLOAD" fi +# When a moe-backend sweep is configured, delegate to its sweep script. +if [[ -n "${WORKLOAD_MOE_BACKENDS:-}" ]]; then + exec "$DIR/run_moe_sweep.sh" "$WORKLOAD" +fi + CONTAINER="perf-eval-${WORKLOAD_NAME}-$$" RESULTS_DIR="results/${WORKLOAD_NAME}" mkdir -p "$RESULTS_DIR" diff --git a/lib/run_moe_sweep.sh b/lib/run_moe_sweep.sh new file mode 100755 index 0000000..046e9af --- /dev/null +++ b/lib/run_moe_sweep.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# MoE-backend sweep: run the full eval suite once per backend listed in +# vllm.moe_backends in the workload YAML. Results land in +# results//moe-/ for each backend. +# +# Invoked by run.sh via exec when WORKLOAD_MOE_BACKENDS is non-empty. +# Can also be run directly: ./lib/run_moe_sweep.sh workloads/foo.yaml +# +# Usage: ./lib/run_moe_sweep.sh + +set -euo pipefail + +WORKLOAD="${1:?usage: $0 }" +[[ -f "$WORKLOAD" ]] || { echo "not found: $WORKLOAD" >&2; exit 2; } + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$DIR/server.sh" +# shellcheck disable=SC1091 +source "$DIR/run_lm_eval.sh" +# shellcheck disable=SC1091 +source "$DIR/run_vllm_bench.sh" +WORKLOAD_EXPORTS="$(python3 "$DIR/parse_workload.py" "$WORKLOAD")" +eval "$WORKLOAD_EXPORTS" +export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME + +PORT=8000 +BASE_URL="http://localhost:${PORT}" +BENCH_TRUST_REMOTE_CODE=false +if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || + [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then + BENCH_TRUST_REMOTE_CODE=true +fi + +mapfile -t MOE_BACKENDS <<< "$WORKLOAD_MOE_BACKENDS" + +for MOE_BACKEND in "${MOE_BACKENDS[@]}"; do + [[ -z "$MOE_BACKEND" ]] && continue + + if [[ "$MOE_BACKEND" == "default" ]]; then + echo "=== :brain: moe backend: (vLLM default)" + RESULTS_DIR="results/${WORKLOAD_NAME}/moe-default" + EFFECTIVE_SERVE_ARGS="$WORKLOAD_SERVE_ARGS" + else + echo "=== :brain: moe backend: ${MOE_BACKEND}" + RESULTS_DIR="results/${WORKLOAD_NAME}/moe-${MOE_BACKEND}" + # --moe-backend is a vLLM server arg, not an env var. + EFFECTIVE_SERVE_ARGS="${WORKLOAD_SERVE_ARGS} --moe-backend ${MOE_BACKEND}" + fi + mkdir -p "$RESULTS_DIR" + + export MOE_BACKEND + + CONTAINER="perf-eval-${WORKLOAD_NAME}-moe-${MOE_BACKEND}-$$" + + trap 'stop_server "$CONTAINER"' EXIT + + if ! start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME"; then + echo "^^^ +++ ERROR: start_server failed for moe backend ${MOE_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if ! wait_healthy "$PORT"; then + echo "^^^ +++ ERROR: vLLM never became healthy for moe backend ${MOE_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if [[ "$MOE_BACKEND" == "default" ]]; then + echo "--- :mag: moe backend selected by vLLM:" + _moe_line="" + if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then + _moe_line=$(grep -oE "Using .+ MoE backend" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null | head -1) || true + else + _moe_line=$(docker logs "$CONTAINER" 2>&1 \ + | grep -oE "Using .+ MoE backend" | head -1) || true + fi + if [[ -n "$_moe_line" ]]; then + echo " $_moe_line" + echo "$_moe_line" > "${RESULTS_DIR}/moe_backend.txt" + # Extract everything between "Using " and " MoE backend" + _resolved=$(echo "$_moe_line" | sed 's/^Using //; s/ MoE backend$//') || true + [[ -n "$_resolved" ]] && export MOE_BACKEND="$_resolved" + else + echo " (moe backend selection line not found in log)" + echo "unknown" > "${RESULTS_DIR}/moe_backend.txt" + fi + else + echo "$MOE_BACKEND" > "${RESULTS_DIR}/moe_backend.txt" + fi + + # vllm bench serve runs first so we can validate perf flow without waiting + # on a full lm_eval pass. Each config's raw json lands in + # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the + # perf dashboard ingest endpoint. + while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do + [[ -z "$bname" ]] && continue + if ! run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR"; then + echo "^^^ +++ ERROR: run_vllm_bench failed for ${bname} (moe backend ${MOE_BACKEND}); skipping run" >&2 + continue + fi + + python3 "$DIR/ingest_perf.py" \ + --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ + --device "$WORKLOAD_BENCH_DEVICE" \ + --tp "$WORKLOAD_BENCH_TP" \ + --precision "$WORKLOAD_BENCH_PRECISION" \ + --model "$WORKLOAD_MODEL" \ + --image "$WORKLOAD_IMAGE" \ + --isl "$isl" --osl "$osl" --conc "$conc" || true + done <<< "$WORKLOAD_VLLM_BENCH_TSV" + + if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + while IFS=$'\t' read -r task fewshot model_args; do + [[ -z "$task" ]] && continue + run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ + "$model_args" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/${task}" \ + --workload "$WORKLOAD_NAME" \ + --task "$task" \ + ${INGEST_NO_SAMPLES:+--no-samples} || true + done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + + # bfcl function-calling eval + while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true + done <<< "$WORKLOAD_BFCL_TSV" + + stop_server "$CONTAINER" + trap - EXIT + drain_gpu +done diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index e8a6240..238b737 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -88,6 +88,7 @@ run_vllm_bench() { mkdir -p "$outdir" local attn_backend="${ATTN_BACKEND:-default}" + local moe_backend="${MOE_BACKEND:-}" local summary_file="${outdir}/bench-${name}-summary.txt" local cmd=(vllm bench serve) @@ -177,7 +178,11 @@ run_vllm_bench() { # Prepend context header to the summary file local _tmp="${summary_file}.tmp" { - echo "attention_backend: ${attn_backend}" + if [[ -n "$moe_backend" ]]; then + echo "moe_backend: ${moe_backend}" + else + echo "attention_backend: ${attn_backend}" + fi echo "isl: ${input_len} osl: ${output_len} conc: ${max_concurrency} n: ${num_prompts}" echo "" cat "$summary_file" diff --git a/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..8521dab --- /dev/null +++ b/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,61 @@ +# GPT-OSS 120B on MI355X +name: moe-sweep-amd-gpt-oss-120b-w-mxfp4-a-fp8-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: amd/gpt-oss-120b-w-mxfp4-a-fp8 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + moe_backends: + - default + - AITER + - AITER_MXFP4_BF16 + #- AITER_MXFP4_FP8 #default 20260626 + #- AITER_MXFP4_MXFP4 + - TRITON + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml index cd2353f..308deab 100644 --- a/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml +++ b/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml @@ -7,7 +7,6 @@ gpu: MI355X num_gpus: 8 nightly: false -#tp should be 8 vllm: model: deepseek-ai/DeepSeek-R1-0528 serve_args: >- @@ -19,7 +18,6 @@ vllm: --max-model-len 10240 --block-size 1 --async-scheduling - --moe-backend AITER --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} env: VLLM_ROCM_USE_AITER: 1 @@ -27,20 +25,20 @@ vllm: NCCL_MIN_NCHANNELS: 112 USE_FASTSAFETENSOR: 1 SAFETENSORS_FAST_GPU: 1 - attention_backends: + moe_backends: - default - #- ROCM_AITER_MLA #default 2026-06-18 - #- ROCM_AITER_TRITON_MLA - #- ROCM_AITER_MLA_SPARSE #not supported Reason: ['non-sparse not supported'] + #- AITER + #- TRITON_UNFUSED + #- MXFP4_BF16 vllm_bench: configs: - # - name: isl1024-osl1024-conc4 - # dataset: random - # input_len: 1024 - # output_len: 1024 - # num_prompts: 200 - # max_concurrency: 4 + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 - name: isl1024-osl1024-conc64 dataset: random input_len: 1024 From 302eb8acb435008cc5f71210aa04cbac44ff9c85 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Fri, 26 Jun 2026 16:08:51 -0500 Subject: [PATCH 52/62] New moe sweep file Signed-off-by: sroberts-amd --- workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml index 8521dab..1363e99 100644 --- a/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml +++ b/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -15,11 +15,11 @@ vllm: --kv-cache-dtype=fp8 moe_backends: - default - - AITER - - AITER_MXFP4_BF16 + #- AITER #only one needed to run, so default + #- AITER_MXFP4_BF16 #- AITER_MXFP4_FP8 #default 20260626 #- AITER_MXFP4_MXFP4 - - TRITON + #- TRITON vllm_bench: configs: From ae4522756cce2afb1e3d0146cea10e35b12cc31a Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Fri, 26 Jun 2026 17:11:14 -0500 Subject: [PATCH 53/62] another updated moe sweep Signed-off-by: sroberts-amd --- .../moe_sweep_deepseek_r1_mxfp4_mi355x.yaml | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml diff --git a/workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml b/workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..a034376 --- /dev/null +++ b/workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,94 @@ +# MoE Backend sweep +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# +name: moe-sweep-deepseek-r1-mxfp4-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + # adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + moe_backends: + - default + #- AITER + #- TRITON_UNFUSED + #- MXFP4_BF16 + + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 From b354ac6d95bfbab3f97871288707b347d4ff42f5 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Mon, 29 Jun 2026 17:19:55 -0500 Subject: [PATCH 54/62] AFO-LLM port, two yaml files, one generate report for this test file. Signed-off-by: sroberts-amd --- gen_scaling_report.py | 640 ++++++++++++++++++ workloads/mi355_ut_gpt_oss_120b_mi355x.yaml | 102 +++ ...i355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml | 101 +++ 3 files changed, 843 insertions(+) create mode 100644 gen_scaling_report.py create mode 100644 workloads/mi355_ut_gpt_oss_120b_mi355x.yaml create mode 100644 workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml diff --git a/gen_scaling_report.py b/gen_scaling_report.py new file mode 100644 index 0000000..f6ed036 --- /dev/null +++ b/gen_scaling_report.py @@ -0,0 +1,640 @@ +#!/usr/bin/env python3 +"""Generate throughput-vs-concurrency scaling reports from perf-eval results. + +One HTML report per (model, ISL/OSL combination), with Chart.js line charts +across concurrency levels — one line per workload variant (e.g. mi355-ut, +vllm-ci). Workload variants that share the same model slug are overlaid on +the same chart set. + +Usage: + python3 gen_scaling_report.py [results_dir] + +Output files are written next to this script: + scaling-.html one per model group + scaling-index.html landing page tabbing between models + +Model grouping: + Result directory names are split on the first '-' that separates a known + prefix (mi355-ut, vllm-ci, upstream) from the model portion. Directories + whose names don't match any known prefix are treated as their own group + with the full name as the label. + + Directories prefixed with attn-sweep- or moe-sweep- are excluded — those + are handled by gen_report.py. + + Example: + mi355-ut-gpt-oss-120b-mi355x → group "gpt-oss-120b-mi355x", label "mi355-ut" + vllm-ci-gpt-oss-120b-mi355x → group "gpt-oss-120b-mi355x", label "vllm-ci" +""" + +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +RESULTS_DIR = Path(__file__).parent / "results" +OUT_DIR = Path(__file__).parent + +# Directories starting with these prefixes belong to gen_report.py, not here. +EXCLUDED_PREFIXES = ("attn-sweep-", "moe-sweep-") + +# Prefixes stripped to derive the model group key. Order matters: longer +# prefixes must come before any prefix that is a prefix of them. +KNOWN_PREFIXES = [ + "mi355-ut-", + "vllm-ci-", + "upstream-", +] + +# ── JSON metric extraction ──────────────────────────────────────────────────── + +METRICS = [ + ("output_throughput", "Output token throughput", "tok/s", True), + ("request_throughput", "Request throughput", "req/s", True), + ("total_token_throughput", "Total token throughput","tok/s", True), + ("mean_ttft_ms", "Mean TTFT", "ms", False), + ("median_ttft_ms", "Median TTFT", "ms", False), + ("p99_ttft_ms", "P99 TTFT", "ms", False), + ("mean_tpot_ms", "Mean TPOT", "ms", False), + ("median_tpot_ms", "Median TPOT", "ms", False), + ("p99_tpot_ms", "P99 TPOT", "ms", False), + ("mean_itl_ms", "Mean ITL", "ms", False), + ("median_itl_ms", "Median ITL", "ms", False), + ("p99_itl_ms", "P99 ITL", "ms", False), +] + +BENCH_NAME_RE = re.compile( + r"bench-isl(?P\d+)-osl(?P\d+)-conc(?P\d+)\.json$" +) + + +def load_result_dir(result_dir: Path) -> dict: + """Return {(isl, osl, conc): {metric_key: float}} for one result directory.""" + points = {} + for jf in sorted(result_dir.glob("bench-isl*-osl*-conc*.json")): + m = BENCH_NAME_RE.search(jf.name) + if not m: + continue + isl, osl, conc = int(m["isl"]), int(m["osl"]), int(m["conc"]) + try: + data = json.loads(jf.read_text()) + except Exception as e: + print(f" warning: could not parse {jf}: {e}", file=sys.stderr) + continue + points[(isl, osl, conc)] = {key: data.get(key) for key, *_ in METRICS} + return points + + +# ── Directory grouping ──────────────────────────────────────────────────────── + +def split_prefix(dir_name: str) -> tuple[str, str]: + """Return (label, model_group) for a result directory name.""" + for prefix in KNOWN_PREFIXES: + if dir_name.startswith(prefix): + return prefix.rstrip("-"), dir_name[len(prefix):] + return dir_name, dir_name + + +def collect_groups(results_dir: Path) -> dict[str, dict[str, dict]]: + """Return {model_group: {label: {(isl,osl,conc): metrics}}}.""" + groups: dict[str, dict] = defaultdict(dict) + for d in sorted(results_dir.iterdir()): + if not d.is_dir(): + continue + if any(d.name.startswith(p) for p in EXCLUDED_PREFIXES): + print(f" skip {d.name} — handled by gen_report.py") + continue + label, group = split_prefix(d.name) + points = load_result_dir(d) + if not points: + print(f" skip {d.name} — no bench JSON files") + continue + # Multiple dirs with the same label in the same group (shouldn't + # normally happen, but merge rather than overwrite) + if label in groups[group]: + groups[group][label].update(points) + else: + groups[group][label] = points + print(f" {d.name} → group={group!r} label={label!r} ({len(points)} points)") + return dict(groups) + + +# ── HTML / JS generation ────────────────────────────────────────────────────── + +# A palette that works for up to ~8 lines; first two match AFO-LLM's colours. +PALETTE = [ + "rgba(102, 126, 234, 1)", # blue-violet (mi355-ut / ATOM) + "rgba(245, 158, 11, 1)", # amber (vllm-ci / ATOM-OOT) + "rgba(244, 114, 182, 1)", # pink (upstream / vLLM) + "rgba( 52, 211, 153, 1)", # emerald (MTP3) + "rgba(251, 191, 36, 1)", # yellow + "rgba(167, 139, 250, 1)", # violet + "rgba( 34, 197, 94, 1)", # green + "rgba(249, 115, 22, 1)", # orange +] + +CSS = """\ +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); + color: #e0e0e0; + min-height: 100vh; + padding: 1.5rem 2rem 3rem; +} + +header { + background: rgba(255,255,255,.05); + backdrop-filter: blur(10px); + padding: 1.5rem 2rem; + border-radius: 12px; + margin-bottom: 1.75rem; + box-shadow: 0 8px 32px rgba(0,0,0,.3); +} + +h1 { + font-size: 1.8rem; + font-weight: 700; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + margin-bottom: .3rem; +} + +.subtitle { color: #94a3b8; font-size: .95rem; } + +/* ISL/OSL tab bar */ +.tab-bar { + display: flex; gap: 0; margin-bottom: 1.5rem; + border-bottom: 1px solid #1e293b; +} +.tab { + padding: .5rem 1.4rem; font-size: .82rem; cursor: pointer; + color: #64748b; border-bottom: 2px solid transparent; + margin-bottom: -1px; user-select: none; transition: color .15s; +} +.tab:hover { color: #94a3b8; } +.tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } +.tab-panel { display: none; } +.tab-panel.active { display: block; } + +/* KPI strip */ +.kpi-strip { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 1rem; margin-bottom: 1.5rem; +} +.kpi-card { + background: rgba(255,255,255,.05); + border: 1px solid rgba(255,255,255,.08); + border-radius: 10px; padding: 1.1rem 1.4rem; + transition: transform .2s, box-shadow .2s; +} +.kpi-card:hover { transform: translateY(-3px); box-shadow: 0 10px 30px rgba(102,126,234,.25); } +.kpi-label { font-size: .8rem; color: #94a3b8; margin-bottom: .4rem; } +.kpi-value { font-size: 1.8rem; font-weight: 700; } +.kpi-sub { font-size: .75rem; color: #64748b; margin-top: .25rem; } +.pos { color: #4ade80; } .neg { color: #f87171; } .neu { color: #60a5fa; } + +/* Chart grid */ +.chart-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); + gap: 1.25rem; margin-bottom: 1.5rem; +} +.chart-card { + background: rgba(255,255,255,.05); + border: 1px solid rgba(255,255,255,.08); + border-radius: 10px; padding: 1.25rem 1.5rem; +} +.chart-card h3 { font-size: 1rem; color: #cbd5e1; margin-bottom: 1rem; } + +/* Detailed table */ +.table-card { + background: rgba(255,255,255,.05); + border: 1px solid rgba(255,255,255,.08); + border-radius: 10px; padding: 1.25rem 1.5rem; + overflow-x: auto; margin-bottom: 1.5rem; +} +.table-card h3 { font-size: 1rem; color: #cbd5e1; margin-bottom: 1rem; } +table { width: 100%; border-collapse: collapse; } +th { + padding: .6rem 1rem; text-align: left; + background: rgba(255,255,255,.08); color: #667eea; + font-size: .8rem; font-weight: 600; cursor: pointer; user-select: none; +} +th:hover { background: rgba(255,255,255,.12); } +td { padding: .55rem 1rem; border-bottom: 1px solid rgba(255,255,255,.05); font-size: .85rem; } +tr:hover td { background: rgba(255,255,255,.03); } +""" + +INDEX_CSS = """\ +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } +html, body { height: 100%; } +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); + color: #e0e0e0; font-size: 14px; + display: flex; flex-direction: column; +} +.header { padding: .75rem 2rem 0; flex-shrink: 0; } +h1 { font-size: 1.2rem; font-weight: 600; color: #f8fafc; margin-bottom: .6rem; } +.model-tab-bar { display: flex; gap: 0; border-bottom: 1px solid #1e293b; } +.model-tab { + padding: .5rem 1.4rem; font-size: .8rem; cursor: pointer; + color: #64748b; border-bottom: 2px solid transparent; + margin-bottom: -1px; user-select: none; white-space: nowrap; transition: color .15s; +} +.model-tab:hover { color: #94a3b8; } +.model-tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } +.frame-container { flex: 1; position: relative; } +iframe { position: absolute; inset: 0; width: 100%; height: 100%; border: none; display: none; } +iframe.active { display: block; } +""" + +INDEX_JS = """\ +function switchModel(el) { + document.querySelectorAll('.model-tab').forEach(t => t.classList.remove('active')); + document.querySelectorAll('iframe').forEach(f => f.classList.remove('active')); + el.classList.add('active'); + var frame = document.getElementById(el.dataset.frame); + frame.classList.add('active'); + if (!frame.src || frame.src === 'about:blank') frame.src = frame.dataset.src; +} +(function() { var first = document.querySelector('.model-tab'); if (first) switchModel(first); })(); +""" + + +def build_html(model_group: str, group_data: dict[str, dict]) -> str: + """Build one scaling report HTML page. + + group_data: {label: {(isl,osl,conc): {metric_key: float|None}}} + """ + labels = sorted(group_data.keys()) + + # Gather all (isl, osl) pairs as tab groups + all_keys: set[tuple] = set() + for pts in group_data.values(): + all_keys.update(pts.keys()) + + isl_osl_pairs = sorted({(isl, osl) for isl, osl, _ in all_keys}) + + # Palette assignment + color_map = {lbl: PALETTE[i % len(PALETTE)] for i, lbl in enumerate(labels)} + # Build JS data blob: { "isl1024-osl1024": { label: { conc: { metric: val } } } } + js_data: dict = {} + for isl, osl in isl_osl_pairs: + group_key = f"isl{isl}-osl{osl}" + js_data[group_key] = {} + for lbl in labels: + pts = group_data[lbl] + conc_map = {} + for (i, o, c), metrics in pts.items(): + if i == isl and o == osl: + conc_map[c] = metrics + if conc_map: + js_data[group_key][lbl] = conc_map + + # Build tab bar + tabs_html = "" + panels_html = "" + for idx, (isl, osl) in enumerate(isl_osl_pairs): + tid = f"isl{isl}-osl{osl}" + active = " active" if idx == 0 else "" + tabs_html += f'
ISL {isl} / OSL {osl}
\n' + panels_html += f'
\n' + + colors_js = json.dumps(color_map) + data_js = json.dumps(js_data) + metrics_js = json.dumps([ + {"key": key, "label": label, "unit": unit, "hi": hi} + for key, label, unit, hi in METRICS + ]) + + js = f""" +const LABELS = {json.dumps(labels)}; +const COLORS = {colors_js}; +const DATA = {data_js}; +const METRICS = {metrics_js}; + +// ── Chart registry ──────────────────────────────────────────────────────────── +const charts = {{}}; + +function destroyCharts(prefix) {{ + Object.keys(charts).filter(k => k.startsWith(prefix)).forEach(k => {{ + charts[k].destroy(); delete charts[k]; + }}); +}} + +// ── KPI strip ───────────────────────────────────────────────────────────────── +function buildKPI(groupKey, container) {{ + const gd = DATA[groupKey]; + if (!gd) return; + + // Use the first label as baseline; pick highest common concurrency + const baseline = LABELS[0]; + const baseConcs = Object.keys(gd[baseline] || {{}}).map(Number); + const conc = baseConcs.includes(64) ? 64 : Math.max(...baseConcs); + + let html = ''; + LABELS.forEach(lbl => {{ + const val = gd[lbl]?.[conc]?.output_throughput; + if (val == null) return; + const baseVal = gd[baseline]?.[conc]?.output_throughput; + let cls = 'neu', delta = ''; + if (lbl !== baseline && baseVal) {{ + const pct = (val - baseVal) / baseVal * 100; + cls = pct >= 0 ? 'pos' : 'neg'; + delta = `
${{pct >= 0 ? '+' : ''}}${{pct.toFixed(1)}}% vs ${{baseline}} @ conc ${{conc}}
`; + }} else {{ + delta = `
@ concurrency ${{conc}}
`; + }} + html += `
+
${{lbl}} — Peak Output Throughput
+
${{val.toFixed(1)}}
+
tok/s
${{delta}} +
`; + }}); + container.insertAdjacentHTML('beforeend', `
${{html}}
`); +}} + +// ── Line chart builder ──────────────────────────────────────────────────────── +function makeChart(canvasId, groupKey, metricKey, title, yLabel) {{ + const gd = DATA[groupKey]; + const concSet = new Set(); + LABELS.forEach(lbl => Object.keys(gd[lbl] || {{}}).forEach(c => concSet.add(Number(c)))); + const concs = Array.from(concSet).sort((a,b) => a - b); + + const datasets = LABELS.map(lbl => ({{ + label: lbl, + data: concs.map(c => gd[lbl]?.[c]?.[metricKey] ?? null), + borderColor: COLORS[lbl], + backgroundColor: COLORS[lbl].replace(', 1)', ', 0.15)'), + borderWidth: 2.5, + pointRadius: 5, + pointHoverRadius: 7, + tension: 0.35, + spanGaps: false, + }})); + + const ctx = document.getElementById(canvasId); + if (!ctx) return; + if (charts[canvasId]) charts[canvasId].destroy(); + charts[canvasId] = new Chart(ctx, {{ + type: 'line', + data: {{ labels: concs, datasets }}, + options: {{ + responsive: true, maintainAspectRatio: true, + plugins: {{ + title: {{ display: false }}, + legend: {{ labels: {{ color: '#cbd5e1', font: {{ size: 12 }} }} }}, + tooltip: {{ + callbacks: {{ + label: ctx => `${{ctx.dataset.label}}: ${{ctx.parsed.y != null ? ctx.parsed.y.toFixed(2) : 'N/A'}} ${{yLabel}}` + }} + }} + }}, + scales: {{ + x: {{ + title: {{ display: true, text: 'Concurrency', color: '#94a3b8' }}, + ticks: {{ color: '#94a3b8' }}, + grid: {{ color: 'rgba(255,255,255,.07)' }}, + }}, + y: {{ + title: {{ display: true, text: yLabel, color: '#94a3b8' }}, + ticks: {{ color: '#94a3b8' }}, + grid: {{ color: 'rgba(255,255,255,.07)' }}, + beginAtZero: false, + }} + }} + }} + }}); +}} + +// ── Chart panels ───────────────────────────────────────────────────────────── +const CHART_GROUPS = [ + {{ + title: 'Throughput', + charts: [ + {{ key: 'output_throughput', label: 'Output Token Throughput', unit: 'tok/s' }}, + {{ key: 'request_throughput', label: 'Request Throughput', unit: 'req/s' }}, + {{ key: 'total_token_throughput', label: 'Total Token Throughput', unit: 'tok/s' }}, + ] + }}, + {{ + title: 'Time to First Token', + charts: [ + {{ key: 'mean_ttft_ms', label: 'Mean TTFT', unit: 'ms' }}, + {{ key: 'median_ttft_ms', label: 'Median TTFT', unit: 'ms' }}, + {{ key: 'p99_ttft_ms', label: 'P99 TTFT', unit: 'ms' }}, + ] + }}, + {{ + title: 'Time per Output Token', + charts: [ + {{ key: 'mean_tpot_ms', label: 'Mean TPOT', unit: 'ms' }}, + {{ key: 'median_tpot_ms', label: 'Median TPOT', unit: 'ms' }}, + {{ key: 'p99_tpot_ms', label: 'P99 TPOT', unit: 'ms' }}, + ] + }}, + {{ + title: 'Inter-token Latency', + charts: [ + {{ key: 'mean_itl_ms', label: 'Mean ITL', unit: 'ms' }}, + {{ key: 'median_itl_ms', label: 'Median ITL', unit: 'ms' }}, + {{ key: 'p99_itl_ms', label: 'P99 ITL', unit: 'ms' }}, + ] + }}, +]; + +function buildCharts(groupKey, container) {{ + CHART_GROUPS.forEach(grp => {{ + let gridHtml = ''; + grp.charts.forEach(c => {{ + const cid = `${{groupKey}}-${{c.key}}`; + gridHtml += `

${{c.label}}

`; + }}); + container.insertAdjacentHTML('beforeend', + `

${{grp.title}}

+
${{gridHtml}}
`); + grp.charts.forEach(c => {{ + makeChart(`${{groupKey}}-${{c.key}}`, groupKey, c.key, c.label, c.unit); + }}); + }}); +}} + +// ── Detailed table ──────────────────────────────────────────────────────────── +function buildTable(groupKey, container) {{ + const gd = DATA[groupKey]; + const concSet = new Set(); + LABELS.forEach(lbl => Object.keys(gd[lbl] || {{}}).forEach(c => concSet.add(Number(c)))); + const concs = Array.from(concSet).sort((a,b) => a - b); + + let rows = ''; + LABELS.forEach(lbl => {{ + concs.forEach(c => {{ + const m = gd[lbl]?.[c]; + if (!m) return; + const f = v => v != null ? v.toFixed(2) : '—'; + rows += ` + ${{lbl}} + ${{c}} + ${{f(m.output_throughput)}} + ${{f(m.request_throughput)}} + ${{f(m.mean_ttft_ms)}} + ${{f(m.mean_tpot_ms)}} + ${{f(m.p99_ttft_ms)}} + ${{f(m.p99_tpot_ms)}} + ${{f(m.mean_itl_ms)}} + `; + }}); + }}); + + container.insertAdjacentHTML('beforeend', ` +
+

Detailed Metrics

+ + + + + + + + + ${{rows}} +
VariantConcurrencyOut tok/sReq/sMean TTFTMean TPOTP99 TTFTP99 TPOTMean ITL
+
`); +}} + +// ── Panel builder (lazy) ────────────────────────────────────────────────────── +function buildPanel(panel) {{ + if (panel.dataset.built) return; + const groupKey = panel.dataset.group; + buildKPI(groupKey, panel); + buildCharts(groupKey, panel); + buildTable(groupKey, panel); + panel.dataset.built = '1'; +}} + +// ── Tab switching ───────────────────────────────────────────────────────────── +function switchTab(el) {{ + document.querySelectorAll('.tab').forEach(t => t.classList.remove('active')); + document.querySelectorAll('.tab-panel').forEach(p => p.classList.remove('active')); + el.classList.add('active'); + const panel = document.getElementById('panel-' + el.dataset.panel); + panel.classList.add('active'); + buildPanel(panel); +}} + +// Build first tab on load +(function() {{ + const firstPanel = document.querySelector('.tab-panel.active'); + if (firstPanel) buildPanel(firstPanel); +}})(); +""" + + html = f""" + + + + + Scaling Report — {model_group} + + + + + +
+

Throughput vs Concurrency

+

Model: {model_group}  ·  Variants: {', '.join(labels)}

+
+ +
+{tabs_html}
+ +{panels_html} + + + + +""" + return html + + +def build_index(reports: list[tuple[str, str]]) -> str: + tabs = "" + frames = "" + for i, (label, fname) in enumerate(reports): + fid = f"frame-{i}" + tabs += f'
{label}
\n' + frames += f' \n' + + return f""" + + + + + Scaling Reports + + + +
+

Throughput vs Concurrency — Scaling Reports

+
+{tabs}
+
+
+{frames}
+ + + +""" + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + results_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else RESULTS_DIR + if not results_dir.exists(): + sys.exit(f"results/ not found at {results_dir}") + + print(f"Scanning {results_dir} …") + groups = collect_groups(results_dir) + if not groups: + sys.exit("No bench JSON files found under results/") + + reports: list[tuple[str, str]] = [] + + for model_group, group_data in sorted(groups.items()): + if not group_data: + continue + html = build_html(model_group, group_data) + fname = f"scaling-{model_group}.html" + (OUT_DIR / fname).write_text(html) + total_pts = sum(len(pts) for pts in group_data.values()) + print(f" wrote {fname} ({len(group_data)} variant(s), {total_pts} total points)") + reports.append((model_group, fname)) + + # Pick up any pre-existing scaling-*.html not produced this run + existing = {fname for _, fname in reports} + for p in sorted(OUT_DIR.glob("scaling-*.html")): + if p.name not in existing and p.name != "scaling-index.html": + reports.append((p.stem.removeprefix("scaling-"), p.name)) + + if reports: + (OUT_DIR / "scaling-index.html").write_text(build_index(reports)) + print(f" wrote scaling-index.html ({len(reports)} model(s))") + + +if __name__ == "__main__": + main() diff --git a/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml b/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..f2370e9 --- /dev/null +++ b/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,102 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# GPT-OSS 120B (FP16 weights, A16W4 fused MoE) on MI355X +# SERVING: TP=1, in=[1024,8192] x out=[1024,8192] x conc=[1,32,64,128] +# ACCURACY: gsm8k 5-shot +# block-size=64 and async-scheduling per mi355_ut extra_args +name: mi355-ut-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 128 + --gpu-memory-utilization 0.95 + --max-model-len 10368 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 + TRITON_HIP_PRESHUFFLE_SCALES: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10368 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl1024-osl8192-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl8192-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl8192-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl8192-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 640 + max_concurrency: 128 diff --git a/workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml b/workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml new file mode 100644 index 0000000..622ddf0 --- /dev/null +++ b/workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml @@ -0,0 +1,101 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# GPT-OSS 120B MXFP4 weights, FP8 KV cache on MI355X +# SERVING: TP=1, in=[1024,8192] x out=[1024,8192] x conc=[1,32,64,128] +# ACCURACY: gsm8k 5-shot +# block-size=64 and async-scheduling per mi355_ut extra_args +name: mi355-ut-gpt-oss-120b-mxfp4-fp8-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: amd/gpt-oss-120b-w-mxfp4-a-fp8 + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 128 + --gpu-memory-utilization 0.95 + --max-model-len 10368 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 + TRITON_HIP_PRESHUFFLE_SCALES: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10368 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl1024-osl8192-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl8192-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl8192-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl8192-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 640 + max_concurrency: 128 From e0636cd3872120840952f3309a437ed9ba30a7c6 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Tue, 30 Jun 2026 09:19:47 -0500 Subject: [PATCH 55/62] adding in deepseek afo-llm port yaml files Signed-off-by: sroberts-amd --- .../mi355_ut_deepseek_r1_0528_mi355x.yaml | 63 +++++++++++++++++++ ..._ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml | 53 ++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml create mode 100644 workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml diff --git a/workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml b/workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..60490ab --- /dev/null +++ b/workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,63 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# DeepSeek-R1-0528 on MI355X (ATOM/OOT env set, no VLLM_USE_V1) +# SERVING: TP=8, in=1024 x out=1024 x conc=[1,32,64] +# ACCURACY: gsm8k 5-shot +# block-size=1 required for MLA; async-scheduling per mi355_ut extra_args +name: mi355-ut-deepseek-r1-0528-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 16384 + --block-size 1 + --async-scheduling + env: + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 16384 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 diff --git a/workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml new file mode 100644 index 0000000..0c18685 --- /dev/null +++ b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml @@ -0,0 +1,53 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# DeepSeek-R1-0528 MXFP4 ASQ checkpoint on MI355X (ATOM/OOT env set, no VLLM_USE_V1) +# SERVING only: TP=8, in=1024 x out=1024 x conc=[1,32,64] +# max_num_seqs=32, max_model_len=70000, kv-cache-dtype=fp8, compilation-config per original +# block-size=1 required for MLA; async-scheduling per mi355_ut extra_args +name: mi355-ut-deepseek-r1-0528-mxfp4-asq-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-0528-MXFP4-ASQ + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 163840 + --max-num-seqs 32 + --gpu-memory-utilization 0.95 + --max-model-len 70000 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"fuse_attn_quant":true,"eliminate_noops":true,"fuse_norm_quant":true},"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"]} + env: + VLLM_DISABLE_COMPILE_CACHE: 1 + AMDGCN_USE_BUFFER_OPS: 1 + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_USE_AITER_MHA: 1 + VLLM_ROCM_USE_AITER_MLA: 0 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 From 352e3982bf07aaf03c48c72dbdae5cc17908626d Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Tue, 30 Jun 2026 14:27:30 -0500 Subject: [PATCH 56/62] fixed server args issue causing bad token id issue. Signed-off-by: sroberts-amd --- ....yaml => mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml} | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) rename workloads/{mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml => mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml} (86%) diff --git a/workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml similarity index 86% rename from workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml rename to workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml index 0c18685..ceba883 100644 --- a/workloads/mi355_ut_deepseek_r1_0528_mxfp4_asq_mi355x.yaml +++ b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml @@ -3,13 +3,13 @@ # SERVING only: TP=8, in=1024 x out=1024 x conc=[1,32,64] # max_num_seqs=32, max_model_len=70000, kv-cache-dtype=fp8, compilation-config per original # block-size=1 required for MLA; async-scheduling per mi355_ut extra_args -name: mi355-ut-deepseek-r1-0528-mxfp4-asq-mi355x +name: mi355-ut-deepseek-r1-0528-mxfp4-mi355x gpu: MI355X num_gpus: 8 nightly: false vllm: - model: amd/DeepSeek-R1-0528-MXFP4-ASQ + model: amd/DeepSeek-R1-MXFP4-Preview #amd/DeepSeek-R1-0528-MXFP4-ASQ serve_args: >- --tensor-parallel-size 8 --dtype auto @@ -20,13 +20,12 @@ vllm: --max-model-len 70000 --block-size 1 --async-scheduling - --compilation-config {"pass_config":{"fuse_attn_quant":true,"eliminate_noops":true,"fuse_norm_quant":true},"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"]} + --compilation-config {"pass_config":{"fuse_attn_quant":true,"eliminate_noops":true,"fuse_norm_quant":true},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"]} env: VLLM_DISABLE_COMPILE_CACHE: 1 AMDGCN_USE_BUFFER_OPS: 1 VLLM_ROCM_USE_AITER: 1 - VLLM_ROCM_USE_AITER_MHA: 1 - VLLM_ROCM_USE_AITER_MLA: 0 + #VLLM_ROCM_USE_AITER_MLA: 0 vllm_bench: configs: From 8482a363bac64d5ef8d8e7159886a1591defc3a5 Mon Sep 17 00:00:00 2001 From: sroberts-amd Date: Tue, 30 Jun 2026 15:13:17 -0500 Subject: [PATCH 57/62] vllm-ci yaml files Signed-off-by: sroberts-amd --- .../vllm_ci_deepseek_r1_0528_mi355x.yaml | 113 +++++++++++++++ workloads/vllm_ci_gpt_oss_120b_mi355x.yaml | 132 ++++++++++++++++++ .../vllm_ci_gpt_oss_120b_tp8_mi355x.yaml | 74 ++++++++++ workloads/vllm_ci_gpt_oss_20b_mi355x.yaml | 128 +++++++++++++++++ 4 files changed, 447 insertions(+) create mode 100644 workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml create mode 100644 workloads/vllm_ci_gpt_oss_120b_mi355x.yaml create mode 100644 workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml create mode 100644 workloads/vllm_ci_gpt_oss_20b_mi355x.yaml diff --git a/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..3d35543 --- /dev/null +++ b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,113 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# DeepSeek-R1-0528 on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# ACCURACY: gsm8k 5-shot +# block-size=1 required for MLA; async-scheduling per vllm_ci extra_args +name: vllm-ci-deepseek-r1-0528-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + env: + VLLM_USE_V1: 1 + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 diff --git a/workloads/vllm_ci_gpt_oss_120b_mi355x.yaml b/workloads/vllm_ci_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..2ebc570 --- /dev/null +++ b/workloads/vllm_ci_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,132 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# GPT-OSS 120B on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=1 and TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64,128] +# ACCURACY: gsm8k 5-shot +# TP=1 and TP=8 configs differ only in the tensor-parallel-size; a single +# workload file covers both since the server is single-instance. For the +# TP=8 case create a separate workload (vllm_ci_gpt_oss_120b_tp8_mi355x.yaml) +# if independent runs are needed. This file covers TP=1 (the more common CI path). +name: vllm-ci-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 diff --git a/workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml b/workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml new file mode 100644 index 0000000..4c26af2 --- /dev/null +++ b/workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml @@ -0,0 +1,74 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# GPT-OSS 120B TP=8 on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16] +# (Lower max concurrency for TP=8 per vllm_ci.yaml) +name: vllm-ci-gpt-oss-120b-tp8-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 diff --git a/workloads/vllm_ci_gpt_oss_20b_mi355x.yaml b/workloads/vllm_ci_gpt_oss_20b_mi355x.yaml new file mode 100644 index 0000000..bcb2efd --- /dev/null +++ b/workloads/vllm_ci_gpt_oss_20b_mi355x.yaml @@ -0,0 +1,128 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# GPT-OSS 20B on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=1, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64,128] +# ACCURACY: gsm8k 5-shot +name: vllm-ci-gpt-oss-20b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-20b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 From 30928103363be6a903f40c46fcaad001c0a16bc1 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Mon, 6 Jul 2026 15:12:26 +0000 Subject: [PATCH 58/62] more report scripts Signed-off-by: Stacy Roberts --- .gitignore | 3 +++ gen_scaling_report.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.gitignore b/.gitignore index 3913b14..c265436 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,9 @@ cython_debug/ # results directory results/ +# test output files +*.txt +*.html # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can diff --git a/gen_scaling_report.py b/gen_scaling_report.py index f6ed036..3792735 100644 --- a/gen_scaling_report.py +++ b/gen_scaling_report.py @@ -15,12 +15,18 @@ Model grouping: Result directory names are split on the first '-' that separates a known +<<<<<<< HEAD prefix (mi355-ut, vllm-ci, upstream) from the model portion. Directories whose names don't match any known prefix are treated as their own group with the full name as the label. Directories prefixed with attn-sweep- or moe-sweep- are excluded — those are handled by gen_report.py. +======= + prefix (mi355-ut, vllm-ci, upstream, attn-sweep, moe-sweep) from the + model portion. Directories whose names don't match any known prefix are + treated as their own group with the full name as the label. +>>>>>>> 79512c283 (more report scripts) Example: mi355-ut-gpt-oss-120b-mi355x → group "gpt-oss-120b-mi355x", label "mi355-ut" @@ -36,12 +42,20 @@ RESULTS_DIR = Path(__file__).parent / "results" OUT_DIR = Path(__file__).parent +<<<<<<< HEAD # Directories starting with these prefixes belong to gen_report.py, not here. EXCLUDED_PREFIXES = ("attn-sweep-", "moe-sweep-") # Prefixes stripped to derive the model group key. Order matters: longer # prefixes must come before any prefix that is a prefix of them. KNOWN_PREFIXES = [ +======= +# Prefixes stripped to derive the model group key. Order matters: longer +# prefixes must come before any prefix that is a prefix of them. +KNOWN_PREFIXES = [ + "attn-sweep-", + "moe-sweep-", +>>>>>>> 79512c283 (more report scripts) "mi355-ut-", "vllm-ci-", "upstream-", @@ -102,9 +116,12 @@ def collect_groups(results_dir: Path) -> dict[str, dict[str, dict]]: for d in sorted(results_dir.iterdir()): if not d.is_dir(): continue +<<<<<<< HEAD if any(d.name.startswith(p) for p in EXCLUDED_PREFIXES): print(f" skip {d.name} — handled by gen_report.py") continue +======= +>>>>>>> 79512c283 (more report scripts) label, group = split_prefix(d.name) points = load_result_dir(d) if not points: From 07998fc1256e9d36875655f7e3f3caebad9dbf85 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Tue, 7 Jul 2026 13:18:44 +0000 Subject: [PATCH 59/62] Fixed memory utilization issue for ds, added in the ds vllm_ci yaml from AFO-LLM port. Signed-off-by: Stacy Roberts Signed-off-by: Stacy Roberts --- ...i355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml | 3 +- workloads/mi355_ut_gpt_oss_120b_mi355x.yaml | 116 +++++++++--------- workloads/moe_sweep_gpt_oss_120b_mi355x.yaml | 71 +++++++++++ .../vllm_ci_deepseek_r1_0528_mi355x.yaml | 2 +- 4 files changed, 131 insertions(+), 61 deletions(-) create mode 100644 workloads/moe_sweep_gpt_oss_120b_mi355x.yaml diff --git a/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml index ceba883..1dd0812 100644 --- a/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml +++ b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml @@ -16,7 +16,7 @@ vllm: --kv-cache-dtype fp8 --max-num-batched-tokens 163840 --max-num-seqs 32 - --gpu-memory-utilization 0.95 + --gpu-memory-utilization 0.92 --max-model-len 70000 --block-size 1 --async-scheduling @@ -25,7 +25,6 @@ vllm: VLLM_DISABLE_COMPILE_CACHE: 1 AMDGCN_USE_BUFFER_OPS: 1 VLLM_ROCM_USE_AITER: 1 - #VLLM_ROCM_USE_AITER_MLA: 0 vllm_bench: configs: diff --git a/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml b/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml index f2370e9..44cedc3 100644 --- a/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml +++ b/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml @@ -42,61 +42,61 @@ lm_eval: max_length: 10368 max_gen_toks: 8192 -vllm_bench: - configs: - - name: isl1024-osl1024-conc1 - backend: openai - dataset: random - input_len: 1024 - output_len: 1024 - num_prompts: 200 - max_concurrency: 1 - - name: isl1024-osl1024-conc32 - backend: openai - dataset: random - input_len: 1024 - output_len: 1024 - num_prompts: 200 - max_concurrency: 32 - - name: isl1024-osl1024-conc64 - backend: openai - dataset: random - input_len: 1024 - output_len: 1024 - num_prompts: 320 - max_concurrency: 64 - - name: isl1024-osl1024-conc128 - backend: openai - dataset: random - input_len: 1024 - output_len: 1024 - num_prompts: 640 - max_concurrency: 128 - - name: isl1024-osl8192-conc1 - backend: openai - dataset: random - input_len: 1024 - output_len: 8192 - num_prompts: 200 - max_concurrency: 1 - - name: isl1024-osl8192-conc32 - backend: openai - dataset: random - input_len: 1024 - output_len: 8192 - num_prompts: 200 - max_concurrency: 32 - - name: isl1024-osl8192-conc64 - backend: openai - dataset: random - input_len: 1024 - output_len: 8192 - num_prompts: 320 - max_concurrency: 64 - - name: isl1024-osl8192-conc128 - backend: openai - dataset: random - input_len: 1024 - output_len: 8192 - num_prompts: 640 - max_concurrency: 128 + #vllm_bench: + # configs: + # - name: isl1024-osl1024-conc1 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 1 + # - name: isl1024-osl1024-conc32 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl1024-osl1024-conc64 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 320 + # max_concurrency: 64 + # - name: isl1024-osl1024-conc128 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 640 + # max_concurrency: 128 + # - name: isl1024-osl8192-conc1 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 200 + # max_concurrency: 1 + # - name: isl1024-osl8192-conc32 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl1024-osl8192-conc64 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 320 + # max_concurrency: 64 + # - name: isl1024-osl8192-conc128 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 640 + # max_concurrency: 128 diff --git a/workloads/moe_sweep_gpt_oss_120b_mi355x.yaml b/workloads/moe_sweep_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..3583634 --- /dev/null +++ b/workloads/moe_sweep_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,71 @@ +# GPT-OSS 120B on MI355X +name: moe-sweep-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + moe_backends: + - default + - TRITON + - AITER + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml index 3d35543..8924286 100644 --- a/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml +++ b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml @@ -15,7 +15,7 @@ vllm: --dtype auto --max-num-batched-tokens 131072 --max-num-seqs 1024 - --gpu-memory-utilization 0.95 + --gpu-memory-utilization 0.92 --max-model-len 10240 --block-size 1 --async-scheduling From 48fbd9822a0ccf65830af2eac084fad8fff2670c Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Tue, 7 Jul 2026 15:11:08 +0000 Subject: [PATCH 60/62] Minor fixes and addition of mxfp4 ds Signed-off-by: Stacy Roberts --- .../vllm_ci_deepseek_r1_0528_mi355x.yaml | 1 - .../vllm_ci_deepseek_r1_mxfp4_mi355x.yaml | 113 ++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml diff --git a/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml index 8924286..ccb4bb5 100644 --- a/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml +++ b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml @@ -20,7 +20,6 @@ vllm: --block-size 1 --async-scheduling env: - VLLM_USE_V1: 1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 VLLM_ROCM_USE_AITER: 1 HSA_NO_SCRATCH_RECLAIM: 1 diff --git a/workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml b/workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..e561bb1 --- /dev/null +++ b/workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,113 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# DeepSeek-R1-0528 on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# ACCURACY: gsm8k 5-shot +# block-size=1 required for MLA; async-scheduling per vllm_ci extra_args +name: vllm-ci-deepseek-r1-mxfp4-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 From 1b10a47516d9d7b9a3713b618776dec99fc67d19 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Tue, 7 Jul 2026 15:11:56 +0000 Subject: [PATCH 61/62] Update to kill child processes (tee was causing issues with trailing processes at end of workflow) Signed-off-by: Stacy Roberts --- lib/server.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/server.sh b/lib/server.sh index 65d24e9..ccee568 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -110,6 +110,7 @@ stop_server() { local container=$1 if [[ -n "${VLLM_LOGS_PID:-}" ]]; then kill "$VLLM_LOGS_PID" 2>/dev/null || true + pkill -P "$VLLM_LOGS_PID" 2>/dev/null || true wait "$VLLM_LOGS_PID" 2>/dev/null || true fi if [[ -n "${VLLM_SERVER_PID:-}" ]]; then From 0155613b5e99993c850e6370a2e21eefda2434d3 Mon Sep 17 00:00:00 2001 From: Stacy Roberts Date: Tue, 14 Jul 2026 13:21:53 +0000 Subject: [PATCH 62/62] Corrections to script plus one configuration addition Signed-off-by: Stacy Roberts --- gen_scaling_report.py | 17 ---------- workloads/deepseek_v32_mi355x.yaml | 52 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 workloads/deepseek_v32_mi355x.yaml diff --git a/gen_scaling_report.py b/gen_scaling_report.py index 3792735..f6ed036 100644 --- a/gen_scaling_report.py +++ b/gen_scaling_report.py @@ -15,18 +15,12 @@ Model grouping: Result directory names are split on the first '-' that separates a known -<<<<<<< HEAD prefix (mi355-ut, vllm-ci, upstream) from the model portion. Directories whose names don't match any known prefix are treated as their own group with the full name as the label. Directories prefixed with attn-sweep- or moe-sweep- are excluded — those are handled by gen_report.py. -======= - prefix (mi355-ut, vllm-ci, upstream, attn-sweep, moe-sweep) from the - model portion. Directories whose names don't match any known prefix are - treated as their own group with the full name as the label. ->>>>>>> 79512c283 (more report scripts) Example: mi355-ut-gpt-oss-120b-mi355x → group "gpt-oss-120b-mi355x", label "mi355-ut" @@ -42,20 +36,12 @@ RESULTS_DIR = Path(__file__).parent / "results" OUT_DIR = Path(__file__).parent -<<<<<<< HEAD # Directories starting with these prefixes belong to gen_report.py, not here. EXCLUDED_PREFIXES = ("attn-sweep-", "moe-sweep-") # Prefixes stripped to derive the model group key. Order matters: longer # prefixes must come before any prefix that is a prefix of them. KNOWN_PREFIXES = [ -======= -# Prefixes stripped to derive the model group key. Order matters: longer -# prefixes must come before any prefix that is a prefix of them. -KNOWN_PREFIXES = [ - "attn-sweep-", - "moe-sweep-", ->>>>>>> 79512c283 (more report scripts) "mi355-ut-", "vllm-ci-", "upstream-", @@ -116,12 +102,9 @@ def collect_groups(results_dir: Path) -> dict[str, dict[str, dict]]: for d in sorted(results_dir.iterdir()): if not d.is_dir(): continue -<<<<<<< HEAD if any(d.name.startswith(p) for p in EXCLUDED_PREFIXES): print(f" skip {d.name} — handled by gen_report.py") continue -======= ->>>>>>> 79512c283 (more report scripts) label, group = split_prefix(d.name) points = load_result_dir(d) if not points: diff --git a/workloads/deepseek_v32_mi355x.yaml b/workloads/deepseek_v32_mi355x.yaml new file mode 100644 index 0000000..20827a4 --- /dev/null +++ b/workloads/deepseek_v32_mi355x.yaml @@ -0,0 +1,52 @@ +# DeepSeek-V3.2 on MI355X +name: deepseek_v32-mi355x +gpu: MI355X +num_gpus: 8 +nightly: true + +vllm: + model: deepseek-ai/DeepSeek-V3.2 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --kv-cache-dtype fp8 + --trust-remote-code + --no-enable-prefix-caching + --gpu-memory-utilization 0.9 + --distributed-executor-backend mp + --speculative_config {"method":"mtp","num_speculative_tokens":3} + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 1k-in-1k-out-conc-32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 512 + max_concurrency: 32 + speed_bench_dataset_subset: throughput_1k + speed_bench_category: low_entropy + - name: 8k-in-1k-out-conc-256 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 256 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy