diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3621a72 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c265436 --- /dev/null +++ b/.gitignore @@ -0,0 +1,167 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# results directory +results/ +# test output files +*.txt +*.html + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/README.md b/README.md index f01a6e6..dc4ecbd 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Each recipe is one `(model, hardware, set of tasks)` combination. The Buildkite workloads/ one YAML per (model, hardware) recipe lib/ orchestrator (run.sh), helpers, GPU profiles .buildkite/ pipeline bootstrap and step generator +gen_report.py generate HTML benchmark reports from results/ CLAUDE.md agent conventions and detailed Buildkite workflow ``` @@ -26,7 +27,7 @@ CLAUDE.md agent conventions and detailed Buildkite workflow A recipe has top-level metadata plus up to three eval blocks: -- **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides). Required. +- **`vllm:`** — *how the server runs.* Defines what model to serve and how (`model`, `serve_args`, optional image/env overrides, optional `attention_backends` or `moe_backends` list). Required. - **`lm_eval:`** — *what accuracy to measure.* Lists lm-evaluation-harness tasks to run against the live server (e.g. `gsm8k`, `aime25`). Each task's score is saved under `results///`. Optional. - **`vllm_bench:`** — *what perf to measure.* Lists `vllm bench serve` configs (input/output lengths, concurrency, dataset). Raw JSON is saved and ingested into the perf dashboard. Optional. - **`bfcl:`** — *function-calling eval.* Runs [BFCL](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) test categories against the live server. Some models need `--enable-auto-tool-choice` and `--tool-call-parser` in `serve_args`. Results are transformed to lm_eval format and ingested as `bfcl_` tasks. Optional. @@ -47,6 +48,13 @@ vllm: # how the server is brought up serve_args: >- # appended to `vllm serve `; word-split -dp 8 --enable-expert-parallel --trust-remote-code + attention_backends: # optional; list of VLLM_ATTENTION_BACKEND values + - FLASH_ATTN # when set, the full eval suite runs once per + - FLASHINFER # backend; results land in attn-/ subdirs + # moe_backends: # optional; list of --moe-backend values + # - default # "default" means no --moe-backend flag (vLLM picks) + # - AITER # results land in moe-/ subdirs + # attention_backends and moe_backends are mutually exclusive in a single workload lm_eval: # accuracy tasks (optional) model_args: # workload-level defaults, merged into every task @@ -85,6 +93,8 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: +- **`vllm.attention_backends`** is an optional list of vLLM attention backend names (`FLASH_ATTN`, `FLASHINFER`, `XFORMERS`, `TRITON_ATTN`, `TRITON_MLA`, `ROCM_FLASH`, `PAGED_ATTENTION`,`ROCM_AITER_FA`,`ROCM_AITER_UNIFIED_ATTN`, `ROCM_ATTN`,`ROCM_AITER_MLA`,`ROCM_AITER_MLA_SPARSE`, `ROCM_AITER_TRITON_MLA`). When set, the orchestrator starts the server once per backend — adding `--attention-backend $ATTN_BACKEND` — and runs the complete eval suite (bench, lm_eval, bfcl) for each. Results are stored under `results//attn-/` so every backend gets its own isolated output directory. Without this field, the server starts once with whatever attention backend vLLM selects by default and results go to `results//` as usual. See `workloads/attn-sweep-gpt-oss-120b-mi355x.yaml` for an example. +- **`vllm.moe_backends`** is an optional list of vLLM `--moe-backend` values (`default`, `AITER`, `TRITON`, `FUSED_MOE`, `deep_gemm_mega_moe`). When set, the orchestrator starts the server once per backend — adding `--moe-backend $MOE_BACKEND` for non-default values — and runs the complete eval suite for each. Results are stored under `results//moe-/`. The special value `"default"` means no `--moe-backend` flag (vLLM picks the backend automatically). **Do not include `--moe-backend` in `serve_args` when using this field** — the sweep script owns that flag. `attention_backends` and `moe_backends` are mutually exclusive in a single workload. See `workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml` for an example. - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. @@ -138,6 +148,22 @@ A real run needs a GPU host with Docker, vLLM, and lm-eval available: Locally, you can smoke-test recipe changes without a GPU — see `CLAUDE.md` for the parser stub and shell-syntax checks. +## Benchmark reports + +After a run completes, generate interactive HTML reports from the `results/` directory: + +```bash +python3 gen_report.py +``` + +This writes one `benchmark-.html` per model directory found under `results/`, plus a `benchmark-index.html` wrapper. Open `benchmark-index.html` in a browser to tab between all models in one page — each model's report loads on demand when its tab is clicked. + +If reports from previous rounds are already present in the directory, `benchmark-index.html` will include them alongside any newly generated ones, so the index always covers every available model regardless of which models were in the current run. + +Each per-model report shows attention backend results side by side, with tabs for each input sequence length, color-coded best/worst values per metric, and percentage deltas relative to the default backend. + +**MoE backend sweep reports** are generated automatically alongside the attention sweep reports. For each model directory that contains `moe-*` result subdirectories, `gen_report.py` writes a `moe-benchmark-.html` file. A `moe-benchmark-index.html` landing page is also produced, covering all models with MoE sweep data. Open it in a browser to tab between models — same layout and features as the attention backend report. + ## Agents `CLAUDE.md` has conventions for AI agents working in this repo: smoke-testing changes, launching Buildkite builds for a chosen branch/commit, and the AI-assistance disclosure rule for PRs and commits. diff --git a/gen_report.py b/gen_report.py new file mode 100644 index 0000000..1488c63 --- /dev/null +++ b/gen_report.py @@ -0,0 +1,839 @@ +#!/usr/bin/env python3 +"""Generate one HTML benchmark report per model directory under ./results, +plus a benchmark-index.html wrapper for tabbing between models.""" + +import os +import re +import sys +import json +from pathlib import Path +from collections import defaultdict + +RESULTS_DIR = Path(__file__).parent / "results" +OUT_DIR = Path(__file__).parent + +# ── Metric extraction ───────────────────────────────────────────────────────── + +METRIC_PATTERNS = { + "Request throughput": r"Request throughput \(req/s\):\s+([\d.]+)", + "Output token throughput": r"Output token throughput \(tok/s\):\s+([\d.]+)", + "Total token throughput": r"Total token throughput \(tok/s\):\s+([\d.]+)", + "Benchmark duration": r"Benchmark duration \(s\):\s+([\d.]+)", + "Mean TTFT": r"Mean TTFT \(ms\):\s+([\d.]+)", + "Median TTFT": r"Median TTFT \(ms\):\s+([\d.]+)", + "P99 TTFT": r"P99 TTFT \(ms\):\s+([\d.]+)", + "Mean TPOT": r"Mean TPOT \(ms\):\s+([\d.]+)", + "Median TPOT": r"Median TPOT \(ms\):\s+([\d.]+)", + "P99 TPOT": r"P99 TPOT \(ms\):\s+([\d.]+)", + "Mean ITL": r"Mean ITL \(ms\):\s+([\d.]+)", + "Median ITL": r"Median ITL \(ms\):\s+([\d.]+)", + "P99 ITL": r"P99 ITL \(ms\):\s+([\d.]+)", +} + +HEADER_PATTERN = re.compile( + r"attention_backend:\s*(\S+).*?isl:\s*(\d+)\s+osl:\s*(\d+)\s+conc:\s*(\d+)", + re.DOTALL, +) +MOE_HEADER_PATTERN = re.compile( + r"moe_backend:\s*(\S+).*?isl:\s*(\d+)\s+osl:\s*(\d+)\s+conc:\s*(\d+)", + re.DOTALL, +) +OVERRIDE_PATTERN = re.compile(r"Overriding with (\S+) out of potential backends") + + +def parse_txt(path: Path) -> dict | None: + text = path.read_text(errors="replace") + m = HEADER_PATTERN.search(text) + if not m: + return None + backend, isl, osl, conc = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4)) + override = OVERRIDE_PATTERN.search(text) + if override: + backend = override.group(1) + metrics = {} + for name, pat in METRIC_PATTERNS.items(): + hit = re.search(pat, text) + metrics[name] = float(hit.group(1)) if hit else None + return {"backend": backend, "isl": isl, "osl": osl, "conc": conc, "metrics": metrics} + + +def parse_txt_moe(path: Path) -> dict | None: + text = path.read_text(errors="replace") + m = MOE_HEADER_PATTERN.search(text) + if not m: + return None + backend, isl, osl, conc = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4)) + metrics = {} + for name, pat in METRIC_PATTERNS.items(): + hit = re.search(pat, text) + metrics[name] = float(hit.group(1)) if hit else None + return {"backend": backend, "isl": isl, "osl": osl, "conc": conc, "metrics": metrics} + + +# ── Directory scan ──────────────────────────────────────────────────────────── + +def collect_model_dir(model_dir: Path) -> tuple[dict, str | None]: + """Return ({backend -> {(isl,osl,conc) -> metrics_dict}}, default_backend_name) + + The default backend is the one whose results live at the top level of the + model directory (not inside an attn-* subdirectory). + """ + data = defaultdict(dict) + default_backend: str | None = None + + def ingest(txt_path: Path, backend_override: str | None = None) -> str | None: + rec = parse_txt(txt_path) + if rec is None: + return None + backend = backend_override or rec["backend"] + key = (rec["isl"], rec["osl"], rec["conc"]) + data[backend][key] = rec["metrics"] + return backend + + # top-level summary files — these are the default backend runs + for txt in sorted(model_dir.glob("*-summary.txt")): + name = ingest(txt) + if name and default_backend is None: + default_backend = name + + # attn-BACKEND subdirs — explicit backend sweep results + for sub in sorted(model_dir.iterdir()): + if sub.is_dir() and sub.name.startswith("attn-"): + backend_name = sub.name[len("attn-"):] + if backend_name == "default": + attn_file = sub / "attn_backend.txt" + if attn_file.exists(): + raw = attn_file.read_text() + m = OVERRIDE_PATTERN.search(raw) + if m: + backend_name = m.group(1) + else: + # Fall back to the last non-empty word on the last line + # (covers "Using XYZ backend" style lines). + last = raw.strip().splitlines()[-1].strip() + backend_name = last.split()[0] if last else "default" + # backend_name may still be "default" if attn_backend.txt is absent. + for txt in sorted(sub.glob("*-summary.txt")): + ingest(txt, backend_override=backend_name) + + return dict(data), default_backend + + +def collect_model_dir_moe(model_dir: Path) -> tuple[dict, str | None]: + """Return ({backend -> {(isl,osl,conc) -> metrics_dict}}, default_backend_name) + + Looks for moe-* subdirectories instead of attn-* subdirectories. + """ + data = defaultdict(dict) + default_backend: str | None = None + + def ingest(txt_path: Path, backend_override: str | None = None) -> str | None: + rec = parse_txt_moe(txt_path) + if rec is None: + return None + backend = backend_override or rec["backend"] + key = (rec["isl"], rec["osl"], rec["conc"]) + data[backend][key] = rec["metrics"] + return backend + + # moe-BACKEND subdirs — explicit moe backend sweep results + for sub in sorted(model_dir.iterdir()): + if sub.is_dir() and sub.name.startswith("moe-"): + backend_name = sub.name[len("moe-"):] + if backend_name == "default": + moe_file = sub / "moe_backend.txt" + if moe_file.exists(): + raw = moe_file.read_text() + # Look for vLLM's moe-specific log line, e.g.: + # Using AITER Fp8 MoE backend out of potential backends: [...] + m = re.search(r"Using (.+?) MoE backend", raw) + if m: + backend_name = m.group(1) + + # backend_name stays "default" if the moe backend line is absent. + for txt in sorted(sub.glob("*-summary.txt")): + name = ingest(txt, backend_override=backend_name) + if name and default_backend is None: + default_backend = name + + return dict(data), default_backend + + +# ── HTML generation ─────────────────────────────────────────────────────────── + +CSS = """ + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + + body { + font-family: ui-monospace, "Cascadia Code", "SF Mono", Menlo, Consolas, monospace; + background: #0f1117; + color: #e2e8f0; + padding: 2rem; + font-size: 14px; + line-height: 1.5; + } + + h1 { + font-size: 1.35rem; + font-weight: 600; + color: #f8fafc; + margin-bottom: 0.3rem; + letter-spacing: -0.01em; + } + + .subtitle { + color: #64748b; + font-size: 0.82rem; + margin-bottom: 1.75rem; + } + + .legend { + display: flex; + gap: 1.5rem; + margin-bottom: 1.5rem; + font-size: 0.75rem; + color: #64748b; + align-items: center; + } + .legend-item { display: flex; align-items: center; gap: 0.4rem; } + .swatch { width: 10px; height: 10px; border-radius: 2px; } + .swatch.green { background: #22c55e; } + .swatch.red { background: #ef4444; } + + .tab-bar { + display: flex; + gap: 0; + margin-bottom: 1.5rem; + border-bottom: 1px solid #1e293b; + } + .tab { + padding: 0.5rem 1.25rem; + font-size: 0.8rem; + cursor: pointer; + color: #64748b; + border-bottom: 2px solid transparent; + margin-bottom: -1px; + user-select: none; + transition: color 0.15s; + } + .tab:hover { color: #94a3b8; } + .tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } + .tab-content { display: none; } + .tab-content.active { display: block; } + + .conc-group { margin-bottom: 2.25rem; } + .conc-label { + font-size: 0.7rem; + font-weight: 700; + color: #475569; + text-transform: uppercase; + letter-spacing: 0.1em; + margin-bottom: 0.6rem; + } + + table { width: 100%; border-collapse: collapse; } + colgroup col:first-child { width: 13rem; } + + th, td { padding: 0.48rem 0.9rem; text-align: right; } + th:first-child, td:first-child { text-align: left; } + + thead tr { border-bottom: 1px solid #1e293b; } + thead th { + font-size: 0.72rem; + font-weight: 500; + color: #64748b; + text-transform: uppercase; + letter-spacing: 0.05em; + padding-bottom: 0.6rem; + } + thead th.backend-name { + font-size: 0.82rem; + text-transform: none; + letter-spacing: normal; + color: #cbd5e1; + font-weight: 600; + } + thead th.backend-name.default-backend { color: #38bdf8; } + + tr.section-row td { + font-size: 0.65rem; + text-transform: uppercase; + letter-spacing: 0.08em; + color: #334155; + padding-top: 0.9rem; + padding-bottom: 0.25rem; + border-bottom: none; + } + + tbody tr { border-bottom: 1px solid #0f172a; } + tbody tr:last-child { border-bottom: none; } + tbody tr:not(.section-row):hover { background: #131c2e; } + + td.metric-label { color: #94a3b8; font-size: 0.8rem; padding-left: 1.5rem; } + td.val { color: #cbd5e1; } + td.val .unit { color: #334155; font-size: 0.68rem; margin-left: 0.2rem; } + td.val.best { color: #22c55e; font-weight: 700; } + td.val.worst { color: #ef4444; } + + .delta { font-size: 0.68rem; margin-left: 0.3rem; opacity: 0.85; } + .delta.good { color: #22c55e; } + .delta.bad { color: #ef4444; } + .delta.neut { color: #475569; } +""" + +JS_TEMPLATE = """ +var BACKENDS = {backends_json}; + +var METRIC_DEFS = [ + {{ label: 'Request throughput', hi: true, unit: 'req/s' }}, + {{ label: 'Output token throughput', hi: true, unit: 'tok/s' }}, + {{ label: 'Total token throughput', hi: true, unit: 'tok/s' }}, + {{ label: 'Benchmark duration', hi: false, unit: 's' }}, +]; +var TTFT_DEFS = [ + {{ label: 'Mean TTFT', hi: false, unit: 'ms' }}, + {{ label: 'Median TTFT', hi: false, unit: 'ms' }}, + {{ label: 'P99 TTFT', hi: false, unit: 'ms' }}, +]; +var TPOT_DEFS = [ + {{ label: 'Mean TPOT', hi: false, unit: 'ms' }}, + {{ label: 'Median TPOT', hi: false, unit: 'ms' }}, + {{ label: 'P99 TPOT', hi: false, unit: 'ms' }}, +]; +var ITL_DEFS = [ + {{ label: 'Mean ITL', hi: false, unit: 'ms' }}, + {{ label: 'Median ITL', hi: false, unit: 'ms' }}, + {{ label: 'P99 ITL', hi: false, unit: 'ms' }}, +]; + +var DATA = {data_json}; + +function fmt(v, unit) {{ + if (v === null || v === undefined) return '—'; + if (unit === 'ms' || unit === 's' || unit === 'req/s' || unit === 'tok/s') return v.toFixed(2); + return String(v); +}} + +function deltaHtml(base, val, hi) {{ + if (base === null || val === null) return ''; + var pct = (val - base) / base * 100; + if (Math.abs(pct) < 0.1) return '(~0%)'; + var sign = pct > 0 ? '+' : ''; + var cls = hi ? (pct > 0 ? 'good' : 'bad') : (pct < 0 ? 'good' : 'bad'); + return '(' + sign + pct.toFixed(1) + '%)'; +}} + +function buildTable(key, concLabel) {{ + var rows = DATA[key]; + if (!rows) return '

No data for ' + key + '

'; + + var sections = [ + {{ label: 'Throughput', defs: METRIC_DEFS }}, + {{ label: 'Time to First Token', defs: TTFT_DEFS }}, + {{ label: 'Time per Output Token', defs: TPOT_DEFS }}, + {{ label: 'Inter-token Latency', defs: ITL_DEFS }}, + ]; + + var headCols = 'Metric'; + BACKENDS.forEach(function(b) {{ + var cls = 'backend-name' + (b.isDefault ? ' default-backend' : ''); + var tag = b.isDefault ? ' (default)' : ''; + headCols += '' + b.label + tag + ''; + }}); + + var bodyHtml = ''; + sections.forEach(function(sec) {{ + bodyHtml += '' + sec.label + ''; + sec.defs.forEach(function(def) {{ + var vals = rows[def.label]; + if (!vals) return; + var valid = vals.map(function(v, i) {{ return {{ v: v, i: i }}; }}).filter(function(x) {{ return x.v !== null; }}); + var bestIdx = null, worstIdx = null; + if (valid.length > 1) {{ + var sorted = valid.slice().sort(function(a, b) {{ return def.hi ? b.v - a.v : a.v - b.v; }}); + bestIdx = sorted[0].i; + worstIdx = sorted[sorted.length - 1].i; + }} + var baseVal = vals[0]; + var cells = '' + def.label + ''; + vals.forEach(function(v, i) {{ + var cls = 'val'; + if (i === bestIdx) cls += ' best'; + if (i === worstIdx) cls += ' worst'; + var delta = i === 0 ? '' : deltaHtml(baseVal, v, def.hi); + cells += '' + fmt(v, def.unit) + '' + def.unit + '' + delta + ''; + }}); + bodyHtml += '' + cells + ''; + }}); + }}); + + return '' + BACKENDS.map(function() {{ return ''; }}).join('') + '' + + '' + headCols + '' + + '' + bodyHtml + '
'; +}} + +function buildContent(islKey) {{ + var concKeys = Object.keys(DATA).filter(function(k) {{ return k.startsWith(islKey + '-'); }}); + concKeys.sort(function(a, b) {{ + var ca = parseInt(a.replace(/.*conc/, '')); + var cb = parseInt(b.replace(/.*conc/, '')); + return ca - cb; + }}); + var html = ''; + concKeys.forEach(function(k) {{ + var conc = k.replace(/.*conc/, ''); + html += '
Concurrency ' + conc + '
' + buildTable(k) + '
'; + }}); + return html || '

No data

'; +}} + +function switchTab(el) {{ + document.querySelectorAll('.tab').forEach(function(t) {{ t.classList.remove('active'); }}); + document.querySelectorAll('.tab-content').forEach(function(c) {{ c.classList.remove('active'); }}); + el.classList.add('active'); + var target = document.getElementById(el.dataset.tab); + target.classList.add('active'); + if (!target.dataset.built) {{ + target.innerHTML = buildContent(el.dataset.tab); + target.dataset.built = '1'; + }} +}} + +// Build first tab on load +(function() {{ + var firstTab = document.querySelector('.tab.active'); + if (firstTab) {{ + var target = document.getElementById(firstTab.dataset.tab); + if (target) {{ + target.innerHTML = buildContent(firstTab.dataset.tab); + target.dataset.built = '1'; + }} + }} +}})(); +""" + + +def build_html(model_name: str, backend_data: dict, default_backend: str | None) -> str: + # Put the default backend first; sort the rest alphabetically + all_backends = sorted(backend_data.keys()) + if default_backend and default_backend in all_backends: + all_backends.remove(default_backend) + all_backends = [default_backend] + all_backends + + # Collect all (isl,osl,conc) test points + all_keys: set[tuple] = set() + for bdata in backend_data.values(): + all_keys.update(bdata.keys()) + + # Build DATA dict: { "isl1024-conc4": { "Mean TTFT": [v_backend0, v_backend1, ...], ... } } + data_dict = {} + for (isl, osl, conc) in sorted(all_keys): + dk = f"isl{isl}-conc{conc}" + row: dict[str, list] = {} + for mname in METRIC_PATTERNS: + vals = [] + for b in all_backends: + bdata = backend_data.get(b, {}) + point = bdata.get((isl, osl, conc), {}) + vals.append(point.get(mname) if point else None) + row[mname] = vals + data_dict[dk] = row + + # Determine ISL groups for tabs + isl_values = sorted({k[0] for k in all_keys}) + osl_values = sorted({k[1] for k in all_keys}) + osl_label = "/".join(str(o) for o in osl_values) + + backends_js = json.dumps([ + {"key": b.lower().replace("_", ""), "label": b, "isDefault": i == 0} + for i, b in enumerate(all_backends) + ], indent=2) + + data_js = json.dumps(data_dict, indent=2) + + # Tab bar HTML + tabs_html = "" + tab_divs = "" + for i, isl in enumerate(isl_values): + tab_id = f"isl{isl}" + active = " active" if i == 0 else "" + tabs_html += f'
ISL {isl} / OSL {osl_label}
\n' + tab_divs += f'
\n' + + subtitle = f"Model: {model_name}" + + html = f""" + + + + + Benchmark — {model_name} + + + + +

Attention Backend Benchmark

+

{subtitle}

+ +
+
Best in row
+
Worst in row
+ · deltas relative to default (first) backend +
+ +
+{tabs_html}
+ +{tab_divs} + + + +""" + return html + + +# ── Index wrapper ───────────────────────────────────────────────────────────── + +INDEX_CSS = """ + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + + html, body { height: 100%; } + + body { + font-family: ui-monospace, "Cascadia Code", "SF Mono", Menlo, Consolas, monospace; + background: #0f1117; + color: #e2e8f0; + font-size: 14px; + display: flex; + flex-direction: column; + } + + .header { + padding: 0.75rem 2rem 0; + flex-shrink: 0; + } + + h1 { + font-size: 1.2rem; + font-weight: 600; + color: #f8fafc; + margin-bottom: 0.6rem; + letter-spacing: -0.01em; + } + + .model-tab-bar { + display: flex; + gap: 0; + border-bottom: 1px solid #1e293b; + } + + .model-tab { + padding: 0.5rem 1.4rem; + font-size: 0.8rem; + cursor: pointer; + color: #64748b; + border-bottom: 2px solid transparent; + margin-bottom: -1px; + user-select: none; + white-space: nowrap; + transition: color 0.15s; + } + .model-tab:hover { color: #94a3b8; } + .model-tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } + + .frame-container { + flex: 1; + position: relative; + } + + iframe { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + border: none; + display: none; + } + iframe.active { display: block; } +""" + +INDEX_JS = """ +function switchModel(el) { + document.querySelectorAll('.model-tab').forEach(function(t) { t.classList.remove('active'); }); + document.querySelectorAll('iframe').forEach(function(f) { f.classList.remove('active'); }); + el.classList.add('active'); + var frame = document.getElementById(el.dataset.frame); + frame.classList.add('active'); + // lazy-load: only set src when first activated + if (!frame.src || frame.src === 'about:blank') { + frame.src = frame.dataset.src; + } +} + +// activate first tab on load +(function() { + var first = document.querySelector('.model-tab'); + if (first) switchModel(first); +})(); +""" + + +def build_index_html(reports: list[tuple[str, str]]) -> str: + """Build a single-page wrapper that tabs between per-model report iframes. + + reports: list of (model_label, filename) sorted as desired. + """ + tabs_html = "" + frames_html = "" + for i, (label, filename) in enumerate(reports): + frame_id = f"frame-{i}" + tabs_html += f'
{label}
\n' + frames_html += f' \n' + + return f""" + + + + + Benchmark Reports + + + + +
+

Attention Backend Benchmarks

+
+{tabs_html}
+
+ +
+{frames_html}
+ + + + +""" + + +def build_html_moe(model_name: str, backend_data: dict, default_backend: str | None) -> str: + """Build a per-model MoE backend benchmark HTML page.""" + all_backends = sorted(backend_data.keys()) + if default_backend and default_backend in all_backends: + all_backends.remove(default_backend) + all_backends = [default_backend] + all_backends + + all_keys: set[tuple] = set() + for bdata in backend_data.values(): + all_keys.update(bdata.keys()) + + data_dict = {} + for (isl, osl, conc) in sorted(all_keys): + dk = f"isl{isl}-conc{conc}" + row: dict[str, list] = {} + for mname in METRIC_PATTERNS: + vals = [] + for b in all_backends: + bdata = backend_data.get(b, {}) + point = bdata.get((isl, osl, conc), {}) + vals.append(point.get(mname) if point else None) + row[mname] = vals + data_dict[dk] = row + + isl_values = sorted({k[0] for k in all_keys}) + osl_values = sorted({k[1] for k in all_keys}) + osl_label = "/".join(str(o) for o in osl_values) + + backends_js = json.dumps([ + {"key": b.lower().replace("_", ""), "label": b, "isDefault": i == 0} + for i, b in enumerate(all_backends) + ], indent=2) + + data_js = json.dumps(data_dict, indent=2) + + tabs_html = "" + tab_divs = "" + for i, isl in enumerate(isl_values): + tab_id = f"isl{isl}" + active = " active" if i == 0 else "" + tabs_html += f'
ISL {isl} / OSL {osl_label}
\n' + tab_divs += f'
\n' + + subtitle = f"Model: {model_name}" + + html = f""" + + + + + MoE Benchmark — {model_name} + + + + +

MoE Backend Benchmark

+

{subtitle}

+ +
+
Best in row
+
Worst in row
+ · deltas relative to default (first) MoE backend +
+ +
+{tabs_html}
+ +{tab_divs} + + + +""" + return html + + +def build_index_html_moe(reports: list[tuple[str, str]]) -> str: + """Build a single-page wrapper that tabs between per-model MoE report iframes.""" + tabs_html = "" + frames_html = "" + for i, (label, filename) in enumerate(reports): + frame_id = f"frame-{i}" + tabs_html += f'
{label}
\n' + frames_html += f' \n' + + return f""" + + + + + MoE Backend Benchmark Reports + + + + +
+

MoE Backend Benchmarks

+
+{tabs_html}
+
+ +
+{frames_html}
+ + + + +""" + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def model_name_from_dir(dir_name: str) -> str: + """Best-effort human label from directory name like attn_sweep_gpt_oss_120b_mi355x.""" + name = re.sub(r"^attn_sweep_", "", dir_name) + return name + + +def model_name_from_dir_moe(dir_name: str) -> str: + """Best-effort human label from directory name like moe_sweep_deepseek_r1_0528_mi355x.""" + name = re.sub(r"^moe_sweep_", "", dir_name) + return name + + +def main(): + if not RESULTS_DIR.exists(): + sys.exit(f"results/ not found at {RESULTS_DIR}") + + model_dirs = [d for d in sorted(RESULTS_DIR.iterdir()) if d.is_dir()] + if not model_dirs: + sys.exit("No subdirectories found in results/") + + reports: list[tuple[str, str]] = [] + + for model_dir in model_dirs: + backend_data, default_backend = collect_model_dir(model_dir) + if not backend_data: + print(f" skip {model_dir.name} — no parseable summary files") + continue + + model_name = model_name_from_dir(model_dir.name) + html = build_html(model_name, backend_data, default_backend) + + out_filename = f"benchmark-{model_dir.name}.html" + out_path = OUT_DIR / out_filename + out_path.write_text(html) + backends = list(backend_data.keys()) + points = sum(len(v) for v in backend_data.values()) + print(f" wrote {out_filename} ({len(backends)} backends, {points} test points)") + reports.append((model_name, out_filename)) + + # Also pick up any pre-existing benchmark-*.html files not produced this run + # (e.g. from a previous round with different models), so the index is complete. + existing = {fname for _, fname in reports} + for html_path in sorted(OUT_DIR.glob("benchmark-*.html")): + if html_path.name not in existing and html_path.name != "benchmark-index.html": + label = model_name_from_dir(html_path.stem.removeprefix("benchmark-")) + reports.append((label, html_path.name)) + + if reports: + index_path = OUT_DIR / "benchmark-index.html" + index_path.write_text(build_index_html(reports)) + print(f" wrote benchmark-index.html ({len(reports)} model(s))") + + # ── MoE backend sweep reports ───────────────────────────────────────────── + moe_reports: list[tuple[str, str]] = [] + + for model_dir in model_dirs: + # Only process directories that contain at least one moe-* subdir + has_moe = any( + sub.is_dir() and sub.name.startswith("moe-") + for sub in model_dir.iterdir() + ) + if not has_moe: + continue + + backend_data, default_backend = collect_model_dir_moe(model_dir) + if not backend_data: + print(f" skip {model_dir.name} (moe) — no parseable summary files") + continue + + model_name = model_name_from_dir_moe(model_dir.name) + html = build_html_moe(model_name, backend_data, default_backend) + + out_filename = f"moe-benchmark-{model_dir.name}.html" + out_path = OUT_DIR / out_filename + out_path.write_text(html) + backends = list(backend_data.keys()) + points = sum(len(v) for v in backend_data.values()) + print(f" wrote {out_filename} ({len(backends)} backends, {points} test points)") + moe_reports.append((model_name, out_filename)) + + # Pick up any pre-existing moe-benchmark-*.html files not produced this run. + existing_moe = {fname for _, fname in moe_reports} + for html_path in sorted(OUT_DIR.glob("moe-benchmark-*.html")): + if html_path.name not in existing_moe and html_path.name != "moe-benchmark-index.html": + label = model_name_from_dir_moe(html_path.stem.removeprefix("moe-benchmark-")) + moe_reports.append((label, html_path.name)) + + if moe_reports: + index_path = OUT_DIR / "moe-benchmark-index.html" + index_path.write_text(build_index_html_moe(moe_reports)) + print(f" wrote moe-benchmark-index.html ({len(moe_reports)} model(s))") + + +if __name__ == "__main__": + main() diff --git a/gen_scaling_report.py b/gen_scaling_report.py new file mode 100644 index 0000000..f6ed036 --- /dev/null +++ b/gen_scaling_report.py @@ -0,0 +1,640 @@ +#!/usr/bin/env python3 +"""Generate throughput-vs-concurrency scaling reports from perf-eval results. + +One HTML report per (model, ISL/OSL combination), with Chart.js line charts +across concurrency levels — one line per workload variant (e.g. mi355-ut, +vllm-ci). Workload variants that share the same model slug are overlaid on +the same chart set. + +Usage: + python3 gen_scaling_report.py [results_dir] + +Output files are written next to this script: + scaling-.html one per model group + scaling-index.html landing page tabbing between models + +Model grouping: + Result directory names are split on the first '-' that separates a known + prefix (mi355-ut, vllm-ci, upstream) from the model portion. Directories + whose names don't match any known prefix are treated as their own group + with the full name as the label. + + Directories prefixed with attn-sweep- or moe-sweep- are excluded — those + are handled by gen_report.py. + + Example: + mi355-ut-gpt-oss-120b-mi355x → group "gpt-oss-120b-mi355x", label "mi355-ut" + vllm-ci-gpt-oss-120b-mi355x → group "gpt-oss-120b-mi355x", label "vllm-ci" +""" + +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +RESULTS_DIR = Path(__file__).parent / "results" +OUT_DIR = Path(__file__).parent + +# Directories starting with these prefixes belong to gen_report.py, not here. +EXCLUDED_PREFIXES = ("attn-sweep-", "moe-sweep-") + +# Prefixes stripped to derive the model group key. Order matters: longer +# prefixes must come before any prefix that is a prefix of them. +KNOWN_PREFIXES = [ + "mi355-ut-", + "vllm-ci-", + "upstream-", +] + +# ── JSON metric extraction ──────────────────────────────────────────────────── + +METRICS = [ + ("output_throughput", "Output token throughput", "tok/s", True), + ("request_throughput", "Request throughput", "req/s", True), + ("total_token_throughput", "Total token throughput","tok/s", True), + ("mean_ttft_ms", "Mean TTFT", "ms", False), + ("median_ttft_ms", "Median TTFT", "ms", False), + ("p99_ttft_ms", "P99 TTFT", "ms", False), + ("mean_tpot_ms", "Mean TPOT", "ms", False), + ("median_tpot_ms", "Median TPOT", "ms", False), + ("p99_tpot_ms", "P99 TPOT", "ms", False), + ("mean_itl_ms", "Mean ITL", "ms", False), + ("median_itl_ms", "Median ITL", "ms", False), + ("p99_itl_ms", "P99 ITL", "ms", False), +] + +BENCH_NAME_RE = re.compile( + r"bench-isl(?P\d+)-osl(?P\d+)-conc(?P\d+)\.json$" +) + + +def load_result_dir(result_dir: Path) -> dict: + """Return {(isl, osl, conc): {metric_key: float}} for one result directory.""" + points = {} + for jf in sorted(result_dir.glob("bench-isl*-osl*-conc*.json")): + m = BENCH_NAME_RE.search(jf.name) + if not m: + continue + isl, osl, conc = int(m["isl"]), int(m["osl"]), int(m["conc"]) + try: + data = json.loads(jf.read_text()) + except Exception as e: + print(f" warning: could not parse {jf}: {e}", file=sys.stderr) + continue + points[(isl, osl, conc)] = {key: data.get(key) for key, *_ in METRICS} + return points + + +# ── Directory grouping ──────────────────────────────────────────────────────── + +def split_prefix(dir_name: str) -> tuple[str, str]: + """Return (label, model_group) for a result directory name.""" + for prefix in KNOWN_PREFIXES: + if dir_name.startswith(prefix): + return prefix.rstrip("-"), dir_name[len(prefix):] + return dir_name, dir_name + + +def collect_groups(results_dir: Path) -> dict[str, dict[str, dict]]: + """Return {model_group: {label: {(isl,osl,conc): metrics}}}.""" + groups: dict[str, dict] = defaultdict(dict) + for d in sorted(results_dir.iterdir()): + if not d.is_dir(): + continue + if any(d.name.startswith(p) for p in EXCLUDED_PREFIXES): + print(f" skip {d.name} — handled by gen_report.py") + continue + label, group = split_prefix(d.name) + points = load_result_dir(d) + if not points: + print(f" skip {d.name} — no bench JSON files") + continue + # Multiple dirs with the same label in the same group (shouldn't + # normally happen, but merge rather than overwrite) + if label in groups[group]: + groups[group][label].update(points) + else: + groups[group][label] = points + print(f" {d.name} → group={group!r} label={label!r} ({len(points)} points)") + return dict(groups) + + +# ── HTML / JS generation ────────────────────────────────────────────────────── + +# A palette that works for up to ~8 lines; first two match AFO-LLM's colours. +PALETTE = [ + "rgba(102, 126, 234, 1)", # blue-violet (mi355-ut / ATOM) + "rgba(245, 158, 11, 1)", # amber (vllm-ci / ATOM-OOT) + "rgba(244, 114, 182, 1)", # pink (upstream / vLLM) + "rgba( 52, 211, 153, 1)", # emerald (MTP3) + "rgba(251, 191, 36, 1)", # yellow + "rgba(167, 139, 250, 1)", # violet + "rgba( 34, 197, 94, 1)", # green + "rgba(249, 115, 22, 1)", # orange +] + +CSS = """\ +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); + color: #e0e0e0; + min-height: 100vh; + padding: 1.5rem 2rem 3rem; +} + +header { + background: rgba(255,255,255,.05); + backdrop-filter: blur(10px); + padding: 1.5rem 2rem; + border-radius: 12px; + margin-bottom: 1.75rem; + box-shadow: 0 8px 32px rgba(0,0,0,.3); +} + +h1 { + font-size: 1.8rem; + font-weight: 700; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + margin-bottom: .3rem; +} + +.subtitle { color: #94a3b8; font-size: .95rem; } + +/* ISL/OSL tab bar */ +.tab-bar { + display: flex; gap: 0; margin-bottom: 1.5rem; + border-bottom: 1px solid #1e293b; +} +.tab { + padding: .5rem 1.4rem; font-size: .82rem; cursor: pointer; + color: #64748b; border-bottom: 2px solid transparent; + margin-bottom: -1px; user-select: none; transition: color .15s; +} +.tab:hover { color: #94a3b8; } +.tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } +.tab-panel { display: none; } +.tab-panel.active { display: block; } + +/* KPI strip */ +.kpi-strip { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 1rem; margin-bottom: 1.5rem; +} +.kpi-card { + background: rgba(255,255,255,.05); + border: 1px solid rgba(255,255,255,.08); + border-radius: 10px; padding: 1.1rem 1.4rem; + transition: transform .2s, box-shadow .2s; +} +.kpi-card:hover { transform: translateY(-3px); box-shadow: 0 10px 30px rgba(102,126,234,.25); } +.kpi-label { font-size: .8rem; color: #94a3b8; margin-bottom: .4rem; } +.kpi-value { font-size: 1.8rem; font-weight: 700; } +.kpi-sub { font-size: .75rem; color: #64748b; margin-top: .25rem; } +.pos { color: #4ade80; } .neg { color: #f87171; } .neu { color: #60a5fa; } + +/* Chart grid */ +.chart-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); + gap: 1.25rem; margin-bottom: 1.5rem; +} +.chart-card { + background: rgba(255,255,255,.05); + border: 1px solid rgba(255,255,255,.08); + border-radius: 10px; padding: 1.25rem 1.5rem; +} +.chart-card h3 { font-size: 1rem; color: #cbd5e1; margin-bottom: 1rem; } + +/* Detailed table */ +.table-card { + background: rgba(255,255,255,.05); + border: 1px solid rgba(255,255,255,.08); + border-radius: 10px; padding: 1.25rem 1.5rem; + overflow-x: auto; margin-bottom: 1.5rem; +} +.table-card h3 { font-size: 1rem; color: #cbd5e1; margin-bottom: 1rem; } +table { width: 100%; border-collapse: collapse; } +th { + padding: .6rem 1rem; text-align: left; + background: rgba(255,255,255,.08); color: #667eea; + font-size: .8rem; font-weight: 600; cursor: pointer; user-select: none; +} +th:hover { background: rgba(255,255,255,.12); } +td { padding: .55rem 1rem; border-bottom: 1px solid rgba(255,255,255,.05); font-size: .85rem; } +tr:hover td { background: rgba(255,255,255,.03); } +""" + +INDEX_CSS = """\ +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } +html, body { height: 100%; } +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); + color: #e0e0e0; font-size: 14px; + display: flex; flex-direction: column; +} +.header { padding: .75rem 2rem 0; flex-shrink: 0; } +h1 { font-size: 1.2rem; font-weight: 600; color: #f8fafc; margin-bottom: .6rem; } +.model-tab-bar { display: flex; gap: 0; border-bottom: 1px solid #1e293b; } +.model-tab { + padding: .5rem 1.4rem; font-size: .8rem; cursor: pointer; + color: #64748b; border-bottom: 2px solid transparent; + margin-bottom: -1px; user-select: none; white-space: nowrap; transition: color .15s; +} +.model-tab:hover { color: #94a3b8; } +.model-tab.active { color: #38bdf8; border-bottom-color: #38bdf8; } +.frame-container { flex: 1; position: relative; } +iframe { position: absolute; inset: 0; width: 100%; height: 100%; border: none; display: none; } +iframe.active { display: block; } +""" + +INDEX_JS = """\ +function switchModel(el) { + document.querySelectorAll('.model-tab').forEach(t => t.classList.remove('active')); + document.querySelectorAll('iframe').forEach(f => f.classList.remove('active')); + el.classList.add('active'); + var frame = document.getElementById(el.dataset.frame); + frame.classList.add('active'); + if (!frame.src || frame.src === 'about:blank') frame.src = frame.dataset.src; +} +(function() { var first = document.querySelector('.model-tab'); if (first) switchModel(first); })(); +""" + + +def build_html(model_group: str, group_data: dict[str, dict]) -> str: + """Build one scaling report HTML page. + + group_data: {label: {(isl,osl,conc): {metric_key: float|None}}} + """ + labels = sorted(group_data.keys()) + + # Gather all (isl, osl) pairs as tab groups + all_keys: set[tuple] = set() + for pts in group_data.values(): + all_keys.update(pts.keys()) + + isl_osl_pairs = sorted({(isl, osl) for isl, osl, _ in all_keys}) + + # Palette assignment + color_map = {lbl: PALETTE[i % len(PALETTE)] for i, lbl in enumerate(labels)} + # Build JS data blob: { "isl1024-osl1024": { label: { conc: { metric: val } } } } + js_data: dict = {} + for isl, osl in isl_osl_pairs: + group_key = f"isl{isl}-osl{osl}" + js_data[group_key] = {} + for lbl in labels: + pts = group_data[lbl] + conc_map = {} + for (i, o, c), metrics in pts.items(): + if i == isl and o == osl: + conc_map[c] = metrics + if conc_map: + js_data[group_key][lbl] = conc_map + + # Build tab bar + tabs_html = "" + panels_html = "" + for idx, (isl, osl) in enumerate(isl_osl_pairs): + tid = f"isl{isl}-osl{osl}" + active = " active" if idx == 0 else "" + tabs_html += f'
ISL {isl} / OSL {osl}
\n' + panels_html += f'
\n' + + colors_js = json.dumps(color_map) + data_js = json.dumps(js_data) + metrics_js = json.dumps([ + {"key": key, "label": label, "unit": unit, "hi": hi} + for key, label, unit, hi in METRICS + ]) + + js = f""" +const LABELS = {json.dumps(labels)}; +const COLORS = {colors_js}; +const DATA = {data_js}; +const METRICS = {metrics_js}; + +// ── Chart registry ──────────────────────────────────────────────────────────── +const charts = {{}}; + +function destroyCharts(prefix) {{ + Object.keys(charts).filter(k => k.startsWith(prefix)).forEach(k => {{ + charts[k].destroy(); delete charts[k]; + }}); +}} + +// ── KPI strip ───────────────────────────────────────────────────────────────── +function buildKPI(groupKey, container) {{ + const gd = DATA[groupKey]; + if (!gd) return; + + // Use the first label as baseline; pick highest common concurrency + const baseline = LABELS[0]; + const baseConcs = Object.keys(gd[baseline] || {{}}).map(Number); + const conc = baseConcs.includes(64) ? 64 : Math.max(...baseConcs); + + let html = ''; + LABELS.forEach(lbl => {{ + const val = gd[lbl]?.[conc]?.output_throughput; + if (val == null) return; + const baseVal = gd[baseline]?.[conc]?.output_throughput; + let cls = 'neu', delta = ''; + if (lbl !== baseline && baseVal) {{ + const pct = (val - baseVal) / baseVal * 100; + cls = pct >= 0 ? 'pos' : 'neg'; + delta = `
${{pct >= 0 ? '+' : ''}}${{pct.toFixed(1)}}% vs ${{baseline}} @ conc ${{conc}}
`; + }} else {{ + delta = `
@ concurrency ${{conc}}
`; + }} + html += `
+
${{lbl}} — Peak Output Throughput
+
${{val.toFixed(1)}}
+
tok/s
${{delta}} +
`; + }}); + container.insertAdjacentHTML('beforeend', `
${{html}}
`); +}} + +// ── Line chart builder ──────────────────────────────────────────────────────── +function makeChart(canvasId, groupKey, metricKey, title, yLabel) {{ + const gd = DATA[groupKey]; + const concSet = new Set(); + LABELS.forEach(lbl => Object.keys(gd[lbl] || {{}}).forEach(c => concSet.add(Number(c)))); + const concs = Array.from(concSet).sort((a,b) => a - b); + + const datasets = LABELS.map(lbl => ({{ + label: lbl, + data: concs.map(c => gd[lbl]?.[c]?.[metricKey] ?? null), + borderColor: COLORS[lbl], + backgroundColor: COLORS[lbl].replace(', 1)', ', 0.15)'), + borderWidth: 2.5, + pointRadius: 5, + pointHoverRadius: 7, + tension: 0.35, + spanGaps: false, + }})); + + const ctx = document.getElementById(canvasId); + if (!ctx) return; + if (charts[canvasId]) charts[canvasId].destroy(); + charts[canvasId] = new Chart(ctx, {{ + type: 'line', + data: {{ labels: concs, datasets }}, + options: {{ + responsive: true, maintainAspectRatio: true, + plugins: {{ + title: {{ display: false }}, + legend: {{ labels: {{ color: '#cbd5e1', font: {{ size: 12 }} }} }}, + tooltip: {{ + callbacks: {{ + label: ctx => `${{ctx.dataset.label}}: ${{ctx.parsed.y != null ? ctx.parsed.y.toFixed(2) : 'N/A'}} ${{yLabel}}` + }} + }} + }}, + scales: {{ + x: {{ + title: {{ display: true, text: 'Concurrency', color: '#94a3b8' }}, + ticks: {{ color: '#94a3b8' }}, + grid: {{ color: 'rgba(255,255,255,.07)' }}, + }}, + y: {{ + title: {{ display: true, text: yLabel, color: '#94a3b8' }}, + ticks: {{ color: '#94a3b8' }}, + grid: {{ color: 'rgba(255,255,255,.07)' }}, + beginAtZero: false, + }} + }} + }} + }}); +}} + +// ── Chart panels ───────────────────────────────────────────────────────────── +const CHART_GROUPS = [ + {{ + title: 'Throughput', + charts: [ + {{ key: 'output_throughput', label: 'Output Token Throughput', unit: 'tok/s' }}, + {{ key: 'request_throughput', label: 'Request Throughput', unit: 'req/s' }}, + {{ key: 'total_token_throughput', label: 'Total Token Throughput', unit: 'tok/s' }}, + ] + }}, + {{ + title: 'Time to First Token', + charts: [ + {{ key: 'mean_ttft_ms', label: 'Mean TTFT', unit: 'ms' }}, + {{ key: 'median_ttft_ms', label: 'Median TTFT', unit: 'ms' }}, + {{ key: 'p99_ttft_ms', label: 'P99 TTFT', unit: 'ms' }}, + ] + }}, + {{ + title: 'Time per Output Token', + charts: [ + {{ key: 'mean_tpot_ms', label: 'Mean TPOT', unit: 'ms' }}, + {{ key: 'median_tpot_ms', label: 'Median TPOT', unit: 'ms' }}, + {{ key: 'p99_tpot_ms', label: 'P99 TPOT', unit: 'ms' }}, + ] + }}, + {{ + title: 'Inter-token Latency', + charts: [ + {{ key: 'mean_itl_ms', label: 'Mean ITL', unit: 'ms' }}, + {{ key: 'median_itl_ms', label: 'Median ITL', unit: 'ms' }}, + {{ key: 'p99_itl_ms', label: 'P99 ITL', unit: 'ms' }}, + ] + }}, +]; + +function buildCharts(groupKey, container) {{ + CHART_GROUPS.forEach(grp => {{ + let gridHtml = ''; + grp.charts.forEach(c => {{ + const cid = `${{groupKey}}-${{c.key}}`; + gridHtml += `

${{c.label}}

`; + }}); + container.insertAdjacentHTML('beforeend', + `

${{grp.title}}

+
${{gridHtml}}
`); + grp.charts.forEach(c => {{ + makeChart(`${{groupKey}}-${{c.key}}`, groupKey, c.key, c.label, c.unit); + }}); + }}); +}} + +// ── Detailed table ──────────────────────────────────────────────────────────── +function buildTable(groupKey, container) {{ + const gd = DATA[groupKey]; + const concSet = new Set(); + LABELS.forEach(lbl => Object.keys(gd[lbl] || {{}}).forEach(c => concSet.add(Number(c)))); + const concs = Array.from(concSet).sort((a,b) => a - b); + + let rows = ''; + LABELS.forEach(lbl => {{ + concs.forEach(c => {{ + const m = gd[lbl]?.[c]; + if (!m) return; + const f = v => v != null ? v.toFixed(2) : '—'; + rows += ` + ${{lbl}} + ${{c}} + ${{f(m.output_throughput)}} + ${{f(m.request_throughput)}} + ${{f(m.mean_ttft_ms)}} + ${{f(m.mean_tpot_ms)}} + ${{f(m.p99_ttft_ms)}} + ${{f(m.p99_tpot_ms)}} + ${{f(m.mean_itl_ms)}} + `; + }}); + }}); + + container.insertAdjacentHTML('beforeend', ` +
+

Detailed Metrics

+ + + + + + + + + ${{rows}} +
VariantConcurrencyOut tok/sReq/sMean TTFTMean TPOTP99 TTFTP99 TPOTMean ITL
+
`); +}} + +// ── Panel builder (lazy) ────────────────────────────────────────────────────── +function buildPanel(panel) {{ + if (panel.dataset.built) return; + const groupKey = panel.dataset.group; + buildKPI(groupKey, panel); + buildCharts(groupKey, panel); + buildTable(groupKey, panel); + panel.dataset.built = '1'; +}} + +// ── Tab switching ───────────────────────────────────────────────────────────── +function switchTab(el) {{ + document.querySelectorAll('.tab').forEach(t => t.classList.remove('active')); + document.querySelectorAll('.tab-panel').forEach(p => p.classList.remove('active')); + el.classList.add('active'); + const panel = document.getElementById('panel-' + el.dataset.panel); + panel.classList.add('active'); + buildPanel(panel); +}} + +// Build first tab on load +(function() {{ + const firstPanel = document.querySelector('.tab-panel.active'); + if (firstPanel) buildPanel(firstPanel); +}})(); +""" + + html = f""" + + + + + Scaling Report — {model_group} + + + + + +
+

Throughput vs Concurrency

+

Model: {model_group}  ·  Variants: {', '.join(labels)}

+
+ +
+{tabs_html}
+ +{panels_html} + + + + +""" + return html + + +def build_index(reports: list[tuple[str, str]]) -> str: + tabs = "" + frames = "" + for i, (label, fname) in enumerate(reports): + fid = f"frame-{i}" + tabs += f'
{label}
\n' + frames += f' \n' + + return f""" + + + + + Scaling Reports + + + +
+

Throughput vs Concurrency — Scaling Reports

+
+{tabs}
+
+
+{frames}
+ + + +""" + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + results_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else RESULTS_DIR + if not results_dir.exists(): + sys.exit(f"results/ not found at {results_dir}") + + print(f"Scanning {results_dir} …") + groups = collect_groups(results_dir) + if not groups: + sys.exit("No bench JSON files found under results/") + + reports: list[tuple[str, str]] = [] + + for model_group, group_data in sorted(groups.items()): + if not group_data: + continue + html = build_html(model_group, group_data) + fname = f"scaling-{model_group}.html" + (OUT_DIR / fname).write_text(html) + total_pts = sum(len(pts) for pts in group_data.values()) + print(f" wrote {fname} ({len(group_data)} variant(s), {total_pts} total points)") + reports.append((model_group, fname)) + + # Pick up any pre-existing scaling-*.html not produced this run + existing = {fname for _, fname in reports} + for p in sorted(OUT_DIR.glob("scaling-*.html")): + if p.name not in existing and p.name != "scaling-index.html": + reports.append((p.stem.removeprefix("scaling-"), p.name)) + + if reports: + (OUT_DIR / "scaling-index.html").write_text(build_index(reports)) + print(f" wrote scaling-index.html ({len(reports)} model(s))") + + +if __name__ == "__main__": + main() diff --git a/lib/ingest_perf.py b/lib/ingest_perf.py index 5291d43..620a7bf 100644 --- a/lib/ingest_perf.py +++ b/lib/ingest_perf.py @@ -49,7 +49,7 @@ def transform(raw: dict, args: argparse.Namespace) -> dict: input_throughput = total_token_throughput - output_throughput data = { - "date": args.date or datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + "date": args.date or datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"), "device": args.device, "conc": int(raw.get("max_concurrency") or args.conc), "image": args.image, diff --git a/lib/parse_workload.py b/lib/parse_workload.py index b6b108e..94e783c 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -20,6 +20,23 @@ import yaml TASK_FIELDS = {"name", "num_fewshot", "model_args"} +VLLM_FIELDS = { + "model", "image", "serve_args", "env", "attention_backends", "moe_backends", +} +KNOWN_ATTENTION_BACKENDS = { + "default", + "FLASH_ATTN", "FLASHINFER", "XFORMERS", "TRITON_ATTN", + "TRITON_MLA", "ROCM_FLASH", "PAGED_ATTENTION", "ROCM_AITER_FA", "ROCM_AITER_MHA", + "ROCM_AITER_UNIFIED_ATTN", "ROCM_ATTN", "ROCM_AITER_MLA", + "ROCM_AITER_MLA_SPARSE", "ROCM_AITER_TRITON_MLA" +} +KNOWN_MOE_BACKENDS = { + "default", + "AITER", "TRITON", "FUSED_MOE", + "AITER_MXFP4_BF16", "AITER_MXFP4_FP8", "TRITON_UNFUSED", + "AITER_MXFP4_MXFP4" +} + BENCH_FIELDS = { "name", "backend", "dataset", "input_len", "output_len", "num_prompts", "max_concurrency", @@ -247,12 +264,47 @@ def validate_bfcl(bfcl: dict, serve_args: str, path: str) -> None: if cat not in BFCL_KNOWN_CATEGORIES: sys.exit(f"{path}: unknown bfcl test category {cat!r}") if "--tool-call-parser" not in serve_args: + print(f"WARNING: {path}: bfcl without --tool-call-parser in serve_args; " + "some models may need it for function-calling", file=sys.stderr) + + +#def bfcl_tsv(bfcl: dict) -> str: +# cats = bfcl.get("test_categories") or [] +# num_threads = bfcl.get("num_threads", 8) +# temperature = bfcl.get("temperature", 0.001) +# return "\n".join(f"{cat}\t{num_threads}\t{temperature}" for cat in cats) + + +def validate_attention_backends(backends: list, path: str) -> None: + if not backends: + sys.exit(f"{path}: vllm.attention_backends must not be empty if specified") + for b in backends: + if b not in KNOWN_ATTENTION_BACKENDS: + sys.exit( + f"{path}: unknown attention backend {b!r}; " + f"known: {', '.join(sorted(KNOWN_ATTENTION_BACKENDS))}" + ) + if len(backends) != len(set(backends)): + sys.exit(f"{path}: duplicate entries in vllm.attention_backends") print( f"WARNING: {path}: bfcl without --tool-call-parser in serve_args; " "some models may need it for function-calling", file=sys.stderr, ) - _validate_bfcl_limits(bfcl, path) + + + +def validate_moe_backends(backends: list, path: str) -> None: + if not backends: + sys.exit(f"{path}: vllm.moe_backends must not be empty if specified") + for b in backends: + if b not in KNOWN_MOE_BACKENDS: + sys.exit( + f"{path}: unknown moe backend {b!r}; " + f"known: {', '.join(sorted(KNOWN_MOE_BACKENDS))}" + ) + if len(backends) != len(set(backends)): + sys.exit(f"{path}: duplicate entries in vllm.moe_backends") def max_test_cases_for_category(bfcl: dict, category: str) -> int | None: @@ -316,6 +368,25 @@ def main(path: str) -> None: if bfcl: validate_bfcl(bfcl, serve_args, path) + attention_backends = vllm.get("attention_backends") or [] + if attention_backends: + validate_attention_backends(attention_backends, path) + + moe_backends = vllm.get("moe_backends") or [] + if moe_backends: + validate_moe_backends(moe_backends, path) + if re.search(r"(^|[\s])--moe-backend(\s|=)", serve_args): + sys.exit( + f"{path}: vllm.moe_backends and --moe-backend in serve_args are mutually exclusive; " + "remove --moe-backend from serve_args when using the moe_backends sweep" + ) + + if attention_backends and moe_backends: + sys.exit( + f"{path}: vllm.attention_backends and vllm.moe_backends are mutually exclusive; " + "run attention and moe sweeps in separate workloads" + ) + image, vllm_commit = resolve_image(vllm, profile) env = {**(profile.get("env") or {}), **(vllm.get("env") or {})} if "HF_HOME" not in env and profile.get("hf_home"): @@ -338,6 +409,11 @@ def main(path: str) -> None: emit("BFCL_TSV", bfcl_tsv(bfcl) if bfcl else "") emit("BENCH_DEVICE", metadata.get("device") or gpu.lower()) emit("BENCH_TP", tp) + #emit("BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or "")) + # One backend per line; empty string when not specified (run.sh treats this + # as a single "default" pass with no override). + emit("ATTENTION_BACKENDS", "\n".join(attention_backends)) + emit("MOE_BACKENDS", "\n".join(moe_backends)) emit( "BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or ""), diff --git a/lib/run.sh b/lib/run.sh index 78f88e5..a12bb23 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -20,14 +20,26 @@ eval "$WORKLOAD_EXPORTS" export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME PORT=8000 -CONTAINER="perf-eval-${WORKLOAD_NAME}-$$" -RESULTS_DIR="results/${WORKLOAD_NAME}" BASE_URL="http://localhost:${PORT}" BENCH_TRUST_REMOTE_CODE=false if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then BENCH_TRUST_REMOTE_CODE=true fi + +# When an attention-backend sweep is configured, delegate entirely to the +# sweep script and exit. Everything below is the single-backend path. +if [[ -n "${WORKLOAD_ATTENTION_BACKENDS:-}" ]]; then + exec "$DIR/run_attn_sweep.sh" "$WORKLOAD" +fi + +# When a moe-backend sweep is configured, delegate to its sweep script. +if [[ -n "${WORKLOAD_MOE_BACKENDS:-}" ]]; then + exec "$DIR/run_moe_sweep.sh" "$WORKLOAD" +fi + +CONTAINER="perf-eval-${WORKLOAD_NAME}-$$" +RESULTS_DIR="results/${WORKLOAD_NAME}" mkdir -p "$RESULTS_DIR" trap 'stop_server "$CONTAINER"' EXIT @@ -75,28 +87,15 @@ while IFS=$'\t' read -r task fewshot model_args; do done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" # bfcl function-calling eval -while IFS=$'\t' read -r category num_threads temperature maximum_step_limit max_test_cases; do +while IFS=$'\t' read -r category num_threads temperature; do [[ -z "$category" ]] && continue echo "--- :phone: bfcl ${category}" python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ - "$category" "$num_threads" "$temperature" "$RESULTS_DIR" \ - "$maximum_step_limit" "$max_test_cases" + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" - manifest="${RESULTS_DIR}/.bfcl_ingest/${category}.txt" - if [[ -f "$manifest" ]]; then - while IFS= read -r ingest_category; do - [[ -z "$ingest_category" ]] && continue - python3 "$DIR/ingest.py" \ - --results-dir "${RESULTS_DIR}/bfcl-${ingest_category}" \ - --workload "$WORKLOAD_NAME" \ - --task "bfcl_${ingest_category}" \ - --no-samples || true - done < "$manifest" - else - python3 "$DIR/ingest.py" \ - --results-dir "${RESULTS_DIR}/bfcl-${category}" \ - --workload "$WORKLOAD_NAME" \ - --task "bfcl_${category}" \ - --no-samples || true - fi + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true done <<< "$WORKLOAD_BFCL_TSV" diff --git a/lib/run_attn_sweep.sh b/lib/run_attn_sweep.sh new file mode 100755 index 0000000..8bd11b7 --- /dev/null +++ b/lib/run_attn_sweep.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Attention-backend sweep: run the full eval suite once per backend listed in +# vllm.attention_backends in the workload YAML. Results land in +# results//attn-/ for each backend. +# +# Invoked by run.sh via exec when WORKLOAD_ATTENTION_BACKENDS is non-empty. +# Can also be run directly: ./lib/run_attn_sweep.sh workloads/foo.yaml +# +# Usage: ./lib/run_attn_sweep.sh + +set -euo pipefail + +WORKLOAD="${1:?usage: $0 }" +[[ -f "$WORKLOAD" ]] || { echo "not found: $WORKLOAD" >&2; exit 2; } + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$DIR/server.sh" +# shellcheck disable=SC1091 +source "$DIR/run_lm_eval.sh" +# shellcheck disable=SC1091 +source "$DIR/run_vllm_bench.sh" +WORKLOAD_EXPORTS="$(python3 "$DIR/parse_workload.py" "$WORKLOAD")" +eval "$WORKLOAD_EXPORTS" +export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME + +PORT=8000 +BASE_URL="http://localhost:${PORT}" +BENCH_TRUST_REMOTE_CODE=false +if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || + [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then + BENCH_TRUST_REMOTE_CODE=true +fi + +mapfile -t ATTN_BACKENDS <<< "$WORKLOAD_ATTENTION_BACKENDS" + +for ATTN_BACKEND in "${ATTN_BACKENDS[@]}"; do + [[ -z "$ATTN_BACKEND" ]] && continue + + if [[ "$ATTN_BACKEND" == "default" ]]; then + echo "=== :brain: attention backend: (vLLM default)" + RESULTS_DIR="results/${WORKLOAD_NAME}/attn-default" + EFFECTIVE_SERVE_ARGS="$WORKLOAD_SERVE_ARGS" + else + echo "=== :brain: attention backend: ${ATTN_BACKEND}" + RESULTS_DIR="results/${WORKLOAD_NAME}/attn-${ATTN_BACKEND}" + # --attention-backend is a vLLM server arg, not an env var. + EFFECTIVE_SERVE_ARGS="${WORKLOAD_SERVE_ARGS} --attention-backend ${ATTN_BACKEND}" + fi + mkdir -p "$RESULTS_DIR" + + CONTAINER="perf-eval-${WORKLOAD_NAME}-${ATTN_BACKEND}-$$" + + trap 'stop_server "$CONTAINER"' EXIT + + if ! start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME"; then + echo "^^^ +++ ERROR: start_server failed for backend ${ATTN_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if ! wait_healthy "$PORT"; then + echo "^^^ +++ ERROR: vLLM never became healthy for backend ${ATTN_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if [[ "$ATTN_BACKEND" == "default" ]]; then + echo "--- :mag: attention backend selected by vLLM:" + _backend_lines="" + if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then + _backend_lines=$(grep -E "(Overriding with|Using [A-Z_]+ backend)" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null) || true + else + _backend_lines=$(docker logs "$CONTAINER" 2>&1 \ + | grep -E "(Overriding with|Using [A-Z_]+ backend)") || true + fi + if [[ -n "$_backend_lines" ]]; then + echo "$_backend_lines" | sed 's/^/ /' + echo "$_backend_lines" > "${RESULTS_DIR}/attn_backend.txt" + else + echo " (backend selection lines not found in log)" + echo "unknown" > "${RESULTS_DIR}/attn_backend.txt" + fi + else + echo "$ATTN_BACKEND" > "${RESULTS_DIR}/attn_backend.txt" + fi + + # vllm bench serve runs first so we can validate perf flow without waiting + # on a full lm_eval pass. Each config's raw json lands in + # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the + # perf dashboard ingest endpoint. + while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do + [[ -z "$bname" ]] && continue + if ! run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR"; then + echo "^^^ +++ ERROR: run_vllm_bench failed for ${bname} (backend ${ATTN_BACKEND}); skipping run" >&2 + continue + fi + + python3 "$DIR/ingest_perf.py" \ + --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ + --device "$WORKLOAD_BENCH_DEVICE" \ + --tp "$WORKLOAD_BENCH_TP" \ + --precision "$WORKLOAD_BENCH_PRECISION" \ + --model "$WORKLOAD_MODEL" \ + --image "$WORKLOAD_IMAGE" \ + --isl "$isl" --osl "$osl" --conc "$conc" || true + done <<< "$WORKLOAD_VLLM_BENCH_TSV" + + if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + while IFS=$'\t' read -r task fewshot model_args; do + [[ -z "$task" ]] && continue + run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ + "$model_args" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/${task}" \ + --workload "$WORKLOAD_NAME" \ + --task "$task" \ + ${INGEST_NO_SAMPLES:+--no-samples} || true + done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + + # bfcl function-calling eval + while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true + done <<< "$WORKLOAD_BFCL_TSV" + + stop_server "$CONTAINER" + trap - EXIT + drain_gpu +done diff --git a/lib/run_moe_sweep.sh b/lib/run_moe_sweep.sh new file mode 100755 index 0000000..046e9af --- /dev/null +++ b/lib/run_moe_sweep.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# MoE-backend sweep: run the full eval suite once per backend listed in +# vllm.moe_backends in the workload YAML. Results land in +# results//moe-/ for each backend. +# +# Invoked by run.sh via exec when WORKLOAD_MOE_BACKENDS is non-empty. +# Can also be run directly: ./lib/run_moe_sweep.sh workloads/foo.yaml +# +# Usage: ./lib/run_moe_sweep.sh + +set -euo pipefail + +WORKLOAD="${1:?usage: $0 }" +[[ -f "$WORKLOAD" ]] || { echo "not found: $WORKLOAD" >&2; exit 2; } + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$DIR/server.sh" +# shellcheck disable=SC1091 +source "$DIR/run_lm_eval.sh" +# shellcheck disable=SC1091 +source "$DIR/run_vllm_bench.sh" +WORKLOAD_EXPORTS="$(python3 "$DIR/parse_workload.py" "$WORKLOAD")" +eval "$WORKLOAD_EXPORTS" +export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME + +PORT=8000 +BASE_URL="http://localhost:${PORT}" +BENCH_TRUST_REMOTE_CODE=false +if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || + [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then + BENCH_TRUST_REMOTE_CODE=true +fi + +mapfile -t MOE_BACKENDS <<< "$WORKLOAD_MOE_BACKENDS" + +for MOE_BACKEND in "${MOE_BACKENDS[@]}"; do + [[ -z "$MOE_BACKEND" ]] && continue + + if [[ "$MOE_BACKEND" == "default" ]]; then + echo "=== :brain: moe backend: (vLLM default)" + RESULTS_DIR="results/${WORKLOAD_NAME}/moe-default" + EFFECTIVE_SERVE_ARGS="$WORKLOAD_SERVE_ARGS" + else + echo "=== :brain: moe backend: ${MOE_BACKEND}" + RESULTS_DIR="results/${WORKLOAD_NAME}/moe-${MOE_BACKEND}" + # --moe-backend is a vLLM server arg, not an env var. + EFFECTIVE_SERVE_ARGS="${WORKLOAD_SERVE_ARGS} --moe-backend ${MOE_BACKEND}" + fi + mkdir -p "$RESULTS_DIR" + + export MOE_BACKEND + + CONTAINER="perf-eval-${WORKLOAD_NAME}-moe-${MOE_BACKEND}-$$" + + trap 'stop_server "$CONTAINER"' EXIT + + if ! start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ + "$EFFECTIVE_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME"; then + echo "^^^ +++ ERROR: start_server failed for moe backend ${MOE_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if ! wait_healthy "$PORT"; then + echo "^^^ +++ ERROR: vLLM never became healthy for moe backend ${MOE_BACKEND}; skipping" >&2 + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + if [[ "$MOE_BACKEND" == "default" ]]; then + echo "--- :mag: moe backend selected by vLLM:" + _moe_line="" + if [[ "${WORKLOAD_SERVER_RUNTIME:-docker}" == "native" ]]; then + _moe_line=$(grep -oE "Using .+ MoE backend" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null | head -1) || true + else + _moe_line=$(docker logs "$CONTAINER" 2>&1 \ + | grep -oE "Using .+ MoE backend" | head -1) || true + fi + if [[ -n "$_moe_line" ]]; then + echo " $_moe_line" + echo "$_moe_line" > "${RESULTS_DIR}/moe_backend.txt" + # Extract everything between "Using " and " MoE backend" + _resolved=$(echo "$_moe_line" | sed 's/^Using //; s/ MoE backend$//') || true + [[ -n "$_resolved" ]] && export MOE_BACKEND="$_resolved" + else + echo " (moe backend selection line not found in log)" + echo "unknown" > "${RESULTS_DIR}/moe_backend.txt" + fi + else + echo "$MOE_BACKEND" > "${RESULTS_DIR}/moe_backend.txt" + fi + + # vllm bench serve runs first so we can validate perf flow without waiting + # on a full lm_eval pass. Each config's raw json lands in + # $RESULTS_DIR/bench-.json and is then transformed and POSTed to the + # perf dashboard ingest endpoint. + while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do + [[ -z "$bname" ]] && continue + if ! run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ + "$conc" "$speed_subset" "$speed_category" \ + "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR"; then + echo "^^^ +++ ERROR: run_vllm_bench failed for ${bname} (moe backend ${MOE_BACKEND}); skipping run" >&2 + continue + fi + + python3 "$DIR/ingest_perf.py" \ + --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ + --device "$WORKLOAD_BENCH_DEVICE" \ + --tp "$WORKLOAD_BENCH_TP" \ + --precision "$WORKLOAD_BENCH_PRECISION" \ + --model "$WORKLOAD_MODEL" \ + --image "$WORKLOAD_IMAGE" \ + --isl "$isl" --osl "$osl" --conc "$conc" || true + done <<< "$WORKLOAD_VLLM_BENCH_TSV" + + if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then + echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" + stop_server "$CONTAINER" + trap - EXIT + drain_gpu + continue + fi + + while IFS=$'\t' read -r task fewshot model_args; do + [[ -z "$task" ]] && continue + run_lm_eval "$WORKLOAD_MODEL" "$BASE_URL" "$task" "$fewshot" \ + "$model_args" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/${task}" \ + --workload "$WORKLOAD_NAME" \ + --task "$task" \ + ${INGEST_NO_SAMPLES:+--no-samples} || true + done <<< "$WORKLOAD_LM_EVAL_TASKS_TSV" + + # bfcl function-calling eval + while IFS=$'\t' read -r category num_threads temperature; do + [[ -z "$category" ]] && continue + echo "--- :phone: bfcl ${category}" + python3 "$DIR/run_bfcl.py" "$WORKLOAD_MODEL" "$BASE_URL" \ + "$category" "$num_threads" "$temperature" "$RESULTS_DIR" + + python3 "$DIR/ingest.py" \ + --results-dir "${RESULTS_DIR}/bfcl-${category}" \ + --workload "$WORKLOAD_NAME" \ + --task "bfcl_${category}" \ + --no-samples || true + done <<< "$WORKLOAD_BFCL_TSV" + + stop_server "$CONTAINER" + trap - EXIT + drain_gpu +done diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index edda073..238b737 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -87,6 +87,10 @@ run_vllm_bench() { echo "--- :stopwatch: vllm bench serve ${name} (dataset=${dataset} isl=${input_len} osl=${output_len} conc=${max_concurrency} n=${num_prompts})" mkdir -p "$outdir" + local attn_backend="${ATTN_BACKEND:-default}" + local moe_backend="${MOE_BACKEND:-}" + local summary_file="${outdir}/bench-${name}-summary.txt" + local cmd=(vllm bench serve) [[ "$runtime" != "native" ]] && cmd=(docker exec "$container" "${cmd[@]}") @@ -140,10 +144,50 @@ run_vllm_bench() { cmd+=(--save-result --result-filename "$in_container_json") fi - "${cmd[@]}" + "${cmd[@]}" | tee "$summary_file" [[ "$runtime" != "native" ]] && docker cp "${container}:${in_container_json}" "$host_json" + # Resolve the actual attention backend selected by vLLM from the live server + # log (server is still running at this point). Named backends are already + # exact; only "default" needs resolution. + if [[ "$attn_backend" == "default" ]]; then + local _resolved + local _log_lines + if [[ "$runtime" == "native" ]]; then + _log_lines=$(grep -E "(Overriding with|Using [A-Z_]+ backend)" \ + "${VLLM_LOG_FILE:-/dev/null}" 2>/dev/null) || true + else + _log_lines=$(docker logs "$container" 2>&1 \ + | grep -E "(Overriding with|Using [A-Z_]+ backend)") || true + fi + # Prefer "Overriding with X" — this is the definitive selection line when + # vLLM discards incompatible backends and picks the actual one used. + _resolved=$(echo "$_log_lines" \ + | grep "Overriding with" | head -1 \ + | grep -oE "Overriding with [A-Z_]+" | awk '{print $3}') || true + # Fall back to "Using X backend" if no override line present + if [[ -z "$_resolved" ]]; then + _resolved=$(echo "$_log_lines" \ + | grep -m1 "Using [A-Z_]+ backend" \ + | grep -oE "Using [A-Z_]+ backend" | awk '{print $2}') || true + fi + [[ -n "$_resolved" ]] && attn_backend="$_resolved" + fi + + # Prepend context header to the summary file + local _tmp="${summary_file}.tmp" + { + if [[ -n "$moe_backend" ]]; then + echo "moe_backend: ${moe_backend}" + else + echo "attention_backend: ${attn_backend}" + fi + echo "isl: ${input_len} osl: ${output_len} conc: ${max_concurrency} n: ${num_prompts}" + echo "" + cat "$summary_file" + } > "$_tmp" && mv "$_tmp" "$summary_file" + python3 - "$host_json" "$num_prompts" <<'PY' import json, sys path, expected = sys.argv[1], int(sys.argv[2]) diff --git a/lib/server.sh b/lib/server.sh index 2d8ce49..ccee568 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -10,6 +10,9 @@ # is also bind-mounted at the same path inside the container so the model cache # on the host is visible to vLLM. For native runtime, values are exported before # starting `vllm serve` in the current job container. +# Note: attention backend selection is passed via --attention-backend in +# serve_args, not as an environment variable (vLLM does not support +# VLLM_ATTENTION_BACKEND as an env var). # # After start_server, vLLM logs are streamed to stdout (prefixed with `[vllm]`) # so build output reflects server startup progress in real time. The streamer's @@ -27,7 +30,12 @@ start_server() { local log_file="/tmp/${container}.log" VLLM_LOG_FILE="$log_file" # shellcheck disable=SC2086 # serve_args intentionally word-split - vllm serve "$model" --port "$port" $serve_args >"$log_file" 2>&1 & + #echo "Server call: vllm server $model --port $port $serve_args" + #vllm serve "$model" --port "$port" $serve_args >"$log_file" 2>&1 & + local -a serve_args_arr + IFS=' ' read -ra serve_args_arr <<< "$serve_args" + echo "Server call: vllm serve $model --port $port ${serve_args_arr[*]}" + vllm serve "$model" --port "$port" "${serve_args_arr[@]}" >"$log_file" 2>&1 & VLLM_SERVER_PID=$! echo "--- :memo: streaming vllm logs" ( tail -f "$log_file" 2>/dev/null | stdbuf -oL -eL sed 's/^/[vllm] /' ) & @@ -49,11 +57,17 @@ start_server() { fi # shellcheck disable=SC2086 # serve_args intentionally word-split + local -a server_args_arr + IFS=' ' read -ra serve_args_arr <<< "$serve_args" # vllm/vllm-openai's entrypoint takes the model as the first positional # arg; do not prepend `vllm` or `serve`. docker run -d --rm --name "$container" "${docker_args[@]}" \ "$image" \ - "$model" --port "$port" $serve_args + # "$model" --port "$port" $serve_args + "$model" --port "$port" "${serve_args_arr[@]}" + + # Install pytest to avoid cupy.testing import failure during torch.compile + docker exec "$container" pip install -q pytest 2>/dev/null || true # Install pytest to avoid cupy.testing import failure during torch.compile docker exec "$container" pip install -q pytest 2>/dev/null || true @@ -96,11 +110,84 @@ stop_server() { local container=$1 if [[ -n "${VLLM_LOGS_PID:-}" ]]; then kill "$VLLM_LOGS_PID" 2>/dev/null || true + pkill -P "$VLLM_LOGS_PID" 2>/dev/null || true wait "$VLLM_LOGS_PID" 2>/dev/null || true fi if [[ -n "${VLLM_SERVER_PID:-}" ]]; then + kill -SIGINT "$VLLM_SERVER_PID" 2>/dev/null || true + wait "$VLLM_SERVER_PID" 2>/dev/null || true kill "$VLLM_SERVER_PID" 2>/dev/null || true wait "$VLLM_SERVER_PID" 2>/dev/null || true fi docker rm -f "$container" >/dev/null 2>&1 || true } + +# Wait until all GPUs have freed their VRAM below a low watermark, or until +# timeout. Call this between backends so the next vllm serve doesn't OOM on +# memory still held by the dying container's ROCm context. +# rocm-smi --showmeminfo vram emits lines like: +# GPU[0] : VRAM Total Used Memory (B): 12345678 +# timeout default: 120sec, threshold_gib default: 0 +drain_gpu() { + local timeout=${1:-120} threshold_gib=${2:-0} + local threshold_bytes=$(( threshold_gib * 1024 * 1024 * 1024 )) + local current_used=0 + echo "--- :hourglass: waiting for GPU VRAM to drain (threshold ${threshold_gib} GiB, timeout ${timeout}s)" + local deadline + deadline=$(( $(date +%s) + timeout )) + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "rocm-smi unavailable; skipping GPU drain check" >&2 + return 0 + fi + while (( $(date +%s) < deadline )); do + current_used=$(rocm-smi --showmeminfo vram --noheader 2>/dev/null \ + | awk '/VRAM Total Used Memory/{if($NF+0>m)m=$NF+0} END{print m+0}') || true + if [[ -z "$current_used" ]]; then + echo "rocm-smi returned no data; skipping GPU drain check" >&2 + return 0 + fi + if (( current_used <= threshold_bytes )); then + echo "GPU VRAM drained (max used: $(( current_used / 1024 / 1024 )) MiB)" + return 0 + fi + echo "GPU still holds $(( current_used / 1024 / 1024 )) MiB VRAM; waiting..." + sleep 5 + done + echo "WARNING: **************************************************************************************************************" >&2 + echo "WARNING: GPU(s) DID NOT DRAIN COMPLETELY " >&2 + echo "WARNING: GPU VRAM did not drain within ${timeout}s (max used: $(( current_used / 1024 / 1024 )) MiB); proceeding anyway" >&2 + echo "WARNING: **************************************************************************************************************" >&2 +} + +# Wait until all GPUs have dropped to 0% utilization, or until timeout. +# Call this after stop_server to confirm the GPU is truly idle before +# starting the next server (complements drain_gpu which checks VRAM). +# rocm-smi --showuse emits lines like: +# GPU[0] : GPU use (%): 12 +# timeout default: 60s +wait_gpu_idle() { + local timeout=${1:-60} + local max_util=0 + echo "--- :hourglass: waiting for GPU utilization to reach 0% (timeout ${timeout}s)" + local deadline + deadline=$(( $(date +%s) + timeout )) + if ! command -v rocm-smi >/dev/null 2>&1; then + echo "rocm-smi unavailable; skipping GPU idle check" >&2 + return 0 + fi + while (( $(date +%s) < deadline )); do + max_util=$(rocm-smi --showuse --noheader 2>/dev/null \ + | awk '/GPU use \(%\)/{if($NF+0>m)m=$NF+0} END{print m+0}') || true + if [[ -z "$max_util" ]]; then + echo "rocm-smi returned no data; skipping GPU idle check" >&2 + return 0 + fi + if (( max_util == 0 )); then + echo "GPU utilization is 0% on all devices" + return 0 + fi + echo "GPU still at ${max_util}% utilization; waiting..." + sleep 5 + done + echo "WARNING: GPU utilization did not reach 0% within ${timeout}s (max: ${max_util}%); proceeding anyway" >&2 +} \ No newline at end of file diff --git a/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..bc784aa --- /dev/null +++ b/workloads/attn_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,74 @@ +# GPT-OSS 120B on MI355X +name: attn-sweep-amd-gpt-oss-120b-w-mxfp4-a-fp8-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: amd/gpt-oss-120b-w-mxfp4-a-fp8 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + attention_backends: + - default + - TRITON_ATTN + #- ROCM_AITER_FA # incompatible in v0.22.0 was default 2026-06-18 + #- ROCM_ATTN #fails for attention sinks not supported + - ROCM_AITER_UNIFIED_ATTN + + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..14cf236 --- /dev/null +++ b/workloads/attn_sweep_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,60 @@ +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: attn-sweep-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + - default + #- ROCM_AITER_MLA #default 2026-06-18 + - ROCM_AITER_TRITON_MLA + #- ROCM_AITER_MLA_SPARSE #not supported Reason: ['non-sparse not supported'] + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 \ No newline at end of file diff --git a/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..cb80109 --- /dev/null +++ b/workloads/attn_sweep_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,94 @@ +# Attention Backend sweep +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# +name: attn-sweep-deepseek-r1-mxfp4-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + # adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + attention_backends: + - default + - ROCM_AITER_TRITON_MLA + - ROCM_AITER_MLA + #- ROCM_AITER_MLA_SPARSE #not supported Reason: ['non-sparse not supported'] + + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml b/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..051ef62 --- /dev/null +++ b/workloads/attn_sweep_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,74 @@ +# GPT-OSS 120B on MI355X +name: attn-sweep-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + attention_backends: + - default + #- ROCM_ATTN #attention sinks not supported + - TRITON_ATTN + #- ROCM_AITER_UNIFIED_ATTN this one is the default + #- ROCM_AITER_FA #fails "attention sinks not supported" + + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..ba9189a --- /dev/null +++ b/workloads/attn_sweep_triton_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,105 @@ +# +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: attn-sweep-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 16 + --async-scheduling + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + #- default #no need to run is run in non-triton config + - TRITON_MLA + +# lm_eval: +# model_args: +# tokenized_requests: false +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml b/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..707ae8d --- /dev/null +++ b/workloads/attn_sweep_triton_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,106 @@ +# +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: attn-sweep-deepseek-r1-mxfp4-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +#tp should be 8 +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 16 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + #- default #no need to run is run in non-triton config + - TRITON_MLA + +# lm_eval: +# model_args: +# tokenized_requests: false +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/deepseek_r1_0528_gsm8k_mi355x.yaml b/workloads/deepseek_r1_0528_gsm8k_mi355x.yaml new file mode 100644 index 0000000..c7af18d --- /dev/null +++ b/workloads/deepseek_r1_0528_gsm8k_mi355x.yaml @@ -0,0 +1,91 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# max_num_seqs=32 (much lower than other groups — intentional per original config). +# extra_args: async-scheduling, kv-cache-dtype=fp8, block-size=1, compilation-config +# The compilation-config JSON is passed as a single quoted string in serve_args. +# No ACCURACY group for this model in vllm_upstream.yaml. +name: deepseek-r1-mxfp4-upstream-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false,},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} +# adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml b/workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml new file mode 100644 index 0000000..97b7529 --- /dev/null +++ b/workloads/deepseek_r1_0528_tritonattnbackends_mi355x.yaml @@ -0,0 +1,105 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: upstream-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 16 + --async-scheduling + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + attention_backends: + - default + - TRITON_MLA #Block size of 16 + +# lm_eval: +# model_args: +# tokenized_requests: false +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl1024-osl1024-conc64 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 64 + # - name: isl8192-osl1024-conc4 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl8192-osl1024-conc64 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 64 diff --git a/workloads/deepseek_v32_mi355x.yaml b/workloads/deepseek_v32_mi355x.yaml new file mode 100644 index 0000000..20827a4 --- /dev/null +++ b/workloads/deepseek_v32_mi355x.yaml @@ -0,0 +1,52 @@ +# DeepSeek-V3.2 on MI355X +name: deepseek_v32-mi355x +gpu: MI355X +num_gpus: 8 +nightly: true + +vllm: + model: deepseek-ai/DeepSeek-V3.2 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --kv-cache-dtype fp8 + --trust-remote-code + --no-enable-prefix-caching + --gpu-memory-utilization 0.9 + --distributed-executor-backend mp + --speculative_config {"method":"mtp","num_speculative_tokens":3} + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 1k-in-1k-out-conc-32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 512 + max_concurrency: 32 + speed_bench_dataset_subset: throughput_1k + speed_bench_category: low_entropy + - name: 8k-in-1k-out-conc-256 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 256 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml new file mode 100644 index 0000000..80077ec --- /dev/null +++ b/workloads/gpt_oss_120b_mi355x_attn_sweep.yaml @@ -0,0 +1,72 @@ +# GPT-OSS 120B on MI355X +name: attn_sweep_gpt_oss_120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + attention_backends: + - default + - ROCM_AITER_UNIFIED_ATTN + - ROCM_AITER_FA + - TRITON_ATTN + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml b/workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..60490ab --- /dev/null +++ b/workloads/mi355_ut_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,63 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# DeepSeek-R1-0528 on MI355X (ATOM/OOT env set, no VLLM_USE_V1) +# SERVING: TP=8, in=1024 x out=1024 x conc=[1,32,64] +# ACCURACY: gsm8k 5-shot +# block-size=1 required for MLA; async-scheduling per mi355_ut extra_args +name: mi355-ut-deepseek-r1-0528-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 16384 + --block-size 1 + --async-scheduling + env: + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 16384 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 diff --git a/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml new file mode 100644 index 0000000..1dd0812 --- /dev/null +++ b/workloads/mi355_ut_deepseek_r1_0528_mxfp4_mi355x.yaml @@ -0,0 +1,51 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# DeepSeek-R1-0528 MXFP4 ASQ checkpoint on MI355X (ATOM/OOT env set, no VLLM_USE_V1) +# SERVING only: TP=8, in=1024 x out=1024 x conc=[1,32,64] +# max_num_seqs=32, max_model_len=70000, kv-cache-dtype=fp8, compilation-config per original +# block-size=1 required for MLA; async-scheduling per mi355_ut extra_args +name: mi355-ut-deepseek-r1-0528-mxfp4-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview #amd/DeepSeek-R1-0528-MXFP4-ASQ + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 163840 + --max-num-seqs 32 + --gpu-memory-utilization 0.92 + --max-model-len 70000 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"fuse_attn_quant":true,"eliminate_noops":true,"fuse_norm_quant":true},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"]} + env: + VLLM_DISABLE_COMPILE_CACHE: 1 + AMDGCN_USE_BUFFER_OPS: 1 + VLLM_ROCM_USE_AITER: 1 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 diff --git a/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml b/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..44cedc3 --- /dev/null +++ b/workloads/mi355_ut_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,102 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# GPT-OSS 120B (FP16 weights, A16W4 fused MoE) on MI355X +# SERVING: TP=1, in=[1024,8192] x out=[1024,8192] x conc=[1,32,64,128] +# ACCURACY: gsm8k 5-shot +# block-size=64 and async-scheduling per mi355_ut extra_args +name: mi355-ut-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 128 + --gpu-memory-utilization 0.95 + --max-model-len 10368 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 + TRITON_HIP_PRESHUFFLE_SCALES: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10368 + max_gen_toks: 8192 + + #vllm_bench: + # configs: + # - name: isl1024-osl1024-conc1 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 1 + # - name: isl1024-osl1024-conc32 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl1024-osl1024-conc64 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 320 + # max_concurrency: 64 + # - name: isl1024-osl1024-conc128 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 640 + # max_concurrency: 128 + # - name: isl1024-osl8192-conc1 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 200 + # max_concurrency: 1 + # - name: isl1024-osl8192-conc32 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 200 + # max_concurrency: 32 + # - name: isl1024-osl8192-conc64 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 320 + # max_concurrency: 64 + # - name: isl1024-osl8192-conc128 + # backend: openai + # dataset: random + # input_len: 1024 + # output_len: 8192 + # num_prompts: 640 + # max_concurrency: 128 diff --git a/workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml b/workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml new file mode 100644 index 0000000..622ddf0 --- /dev/null +++ b/workloads/mi355_ut_gpt_oss_120b_mxfp4_fp8_mi355x.yaml @@ -0,0 +1,101 @@ +# Ported from AFO-LLM configs/mi355_ut.yaml +# GPT-OSS 120B MXFP4 weights, FP8 KV cache on MI355X +# SERVING: TP=1, in=[1024,8192] x out=[1024,8192] x conc=[1,32,64,128] +# ACCURACY: gsm8k 5-shot +# block-size=64 and async-scheduling per mi355_ut extra_args +name: mi355-ut-gpt-oss-120b-mxfp4-fp8-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: amd/gpt-oss-120b-w-mxfp4-a-fp8 + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 128 + --gpu-memory-utilization 0.95 + --max-model-len 10368 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 + TRITON_HIP_PRESHUFFLE_SCALES: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10368 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl1024-osl8192-conc1 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 200 + max_concurrency: 1 + - name: isl1024-osl8192-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl8192-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl8192-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 8192 + num_prompts: 640 + max_concurrency: 128 diff --git a/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml b/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..1363e99 --- /dev/null +++ b/workloads/moe_sweep_amd_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,61 @@ +# GPT-OSS 120B on MI355X +name: moe-sweep-amd-gpt-oss-120b-w-mxfp4-a-fp8-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: amd/gpt-oss-120b-w-mxfp4-a-fp8 + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + moe_backends: + - default + #- AITER #only one needed to run, so default + #- AITER_MXFP4_BF16 + #- AITER_MXFP4_FP8 #default 20260626 + #- AITER_MXFP4_MXFP4 + #- TRITON + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml b/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..308deab --- /dev/null +++ b/workloads/moe_sweep_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,59 @@ +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: moe-sweep-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + moe_backends: + - default + #- AITER + #- TRITON_UNFUSED + #- MXFP4_BF16 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 \ No newline at end of file diff --git a/workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml b/workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..a034376 --- /dev/null +++ b/workloads/moe_sweep_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,94 @@ +# MoE Backend sweep +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# +name: moe-sweep-deepseek-r1-mxfp4-MI355X +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + # adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + moe_backends: + - default + #- AITER + #- TRITON_UNFUSED + #- MXFP4_BF16 + + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl1024-osl1024-conc8 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl1024-osl1024-conc16 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl1024-osl1024-conc32 + # dataset: random + # input_len: 1024 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + # - name: isl8192-osl1024-conc8 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 8 + # - name: isl8192-osl1024-conc16 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 16 + # - name: isl8192-osl1024-conc32 + # dataset: random + # input_len: 8192 + # output_len: 1024 + # num_prompts: 200 + # max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/moe_sweep_gpt_oss_120b_mi355x.yaml b/workloads/moe_sweep_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..3583634 --- /dev/null +++ b/workloads/moe_sweep_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,71 @@ +# GPT-OSS 120B on MI355X +name: moe-sweep-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 8 +# Opt-in for now: WORKLOADS=gpt_oss_120b_mi355x +nightly: false + +vllm: + model: openai/gpt-oss-120b + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 1 + --trust-remote-code + --kv-cache-dtype=fp8 + moe_backends: + - default + - TRITON + - AITER + +# lm_eval: +# model_args: +# tokenized_requests: false +# tokenizer_backend: null +# timeout: 6000 +# tasks: +# - name: gsm8k +# num_fewshot: 5 +# model_args: +# num_concurrent: 64 +# max_length: 32768 +# max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/upstream_deepseek_r1_0528_mi355x.yaml b/workloads/upstream_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..d8fd942 --- /dev/null +++ b/workloads/upstream_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,103 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# ACCURACY (TP=8, block_size=1) + SERVING (TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64]) +# block-size=1 is a DeepSeek MLA requirement; passed via serve_args for both roles. +# async-scheduling is a SERVING extra_arg in the original; included here since the +# same server instance handles both eval types. +name: upstream-deepseek-r1-0528-MI355X +gpu: MI355X +num_gpus: 1 #8 +nightly: false + +#tp should be 8 +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + env: +# VLLM_USE_V1: 1 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml b/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..c7af18d --- /dev/null +++ b/workloads/upstream_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,91 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# SERVING only — TP=8, MXFP4 preview checkpoint +# in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# max_num_seqs=32 (much lower than other groups — intentional per original config). +# extra_args: async-scheduling, kv-cache-dtype=fp8, block-size=1, compilation-config +# The compilation-config JSON is passed as a single quoted string in serve_args. +# No ACCURACY group for this model in vllm_upstream.yaml. +name: deepseek-r1-mxfp4-upstream-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + env: + VLLM_ROCM_USE_AITER: 1 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --kv-cache-dtype fp8 + --max-num-batched-tokens 131072 + --max-num-seqs 32 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false,},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} +# adding this to compilation-config errors on unrecognized flags "pass_config":{"enable_attn_fusion":true,"enable_noop":true,"enable_fusion":true}, + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 diff --git a/workloads/upstream_gpt_oss_20b_mi355x.yaml b/workloads/upstream_gpt_oss_20b_mi355x.yaml new file mode 100644 index 0000000..63209b5 --- /dev/null +++ b/workloads/upstream_gpt_oss_20b_mi355x.yaml @@ -0,0 +1,116 @@ +# Ported from AFO-LLM configs/vllm_upstream.yaml +# ACCURACY (TP=1) + SERVING (TP=1, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64,128]) +# extra_args block-size=64 and async-scheduling go into serve_args. +# ACCURACY uses lm_eval (gsm8k 5-shot, matching other upstream workloads). +name: upstream-gpt-oss-20b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-20b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 +attention_backends: + - ROCM_AITER_UNIFIED_ATTN + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 128 diff --git a/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml new file mode 100644 index 0000000..ccb4bb5 --- /dev/null +++ b/workloads/vllm_ci_deepseek_r1_0528_mi355x.yaml @@ -0,0 +1,112 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# DeepSeek-R1-0528 on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# ACCURACY: gsm8k 5-shot +# block-size=1 required for MLA; async-scheduling per vllm_ci extra_args +name: vllm-ci-deepseek-r1-0528-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: deepseek-ai/DeepSeek-R1-0528 + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + env: + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 diff --git a/workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml b/workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml new file mode 100644 index 0000000..e561bb1 --- /dev/null +++ b/workloads/vllm_ci_deepseek_r1_mxfp4_mi355x.yaml @@ -0,0 +1,113 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# DeepSeek-R1-0528 on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64] +# ACCURACY: gsm8k 5-shot +# block-size=1 required for MLA; async-scheduling per vllm_ci extra_args +name: vllm-ci-deepseek-r1-mxfp4-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: amd/DeepSeek-R1-MXFP4-Preview + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 131072 + --max-num-seqs 1024 + --gpu-memory-utilization 0.92 + --max-model-len 10240 + --block-size 1 + --async-scheduling + --compilation-config {"pass_config":{"eliminate_noops":false},"cudagraph_mode":"FULL","custom_ops":["+rms_norm","+silu_and_mul","+quant_fp8"],"splitting_ops":[]} + env: + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER: 1 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 diff --git a/workloads/vllm_ci_gpt_oss_120b_mi355x.yaml b/workloads/vllm_ci_gpt_oss_120b_mi355x.yaml new file mode 100644 index 0000000..2ebc570 --- /dev/null +++ b/workloads/vllm_ci_gpt_oss_120b_mi355x.yaml @@ -0,0 +1,132 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# GPT-OSS 120B on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=1 and TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64,128] +# ACCURACY: gsm8k 5-shot +# TP=1 and TP=8 configs differ only in the tensor-parallel-size; a single +# workload file covers both since the server is single-instance. For the +# TP=8 case create a separate workload (vllm_ci_gpt_oss_120b_tp8_mi355x.yaml) +# if independent runs are needed. This file covers TP=1 (the more common CI path). +name: vllm-ci-gpt-oss-120b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 diff --git a/workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml b/workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml new file mode 100644 index 0000000..4c26af2 --- /dev/null +++ b/workloads/vllm_ci_gpt_oss_120b_tp8_mi355x.yaml @@ -0,0 +1,74 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# GPT-OSS 120B TP=8 on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=8, in=[1024,8192] x out=1024 x conc=[4,8,16] +# (Lower max concurrency for TP=8 per vllm_ci.yaml) +name: vllm-ci-gpt-oss-120b-tp8-mi355x +gpu: MI355X +num_gpus: 8 +nightly: false + +vllm: + model: openai/gpt-oss-120b + serve_args: >- + --tensor-parallel-size 8 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 diff --git a/workloads/vllm_ci_gpt_oss_20b_mi355x.yaml b/workloads/vllm_ci_gpt_oss_20b_mi355x.yaml new file mode 100644 index 0000000..bcb2efd --- /dev/null +++ b/workloads/vllm_ci_gpt_oss_20b_mi355x.yaml @@ -0,0 +1,128 @@ +# Ported from AFO-LLM configs/vllm_ci.yaml +# GPT-OSS 20B on MI355X with upstream vLLM (VLLM_USE_V1=1) +# SERVING: TP=1, in=[1024,8192] x out=1024 x conc=[4,8,16,32,64,128] +# ACCURACY: gsm8k 5-shot +name: vllm-ci-gpt-oss-20b-mi355x +gpu: MI355X +num_gpus: 1 +nightly: false + +vllm: + model: openai/gpt-oss-20b + serve_args: >- + --tensor-parallel-size 1 + --dtype auto + --max-num-batched-tokens 8192 + --max-num-seqs 1024 + --gpu-memory-utilization 0.95 + --max-model-len 10240 + --block-size 64 + --async-scheduling + --trust-remote-code + env: + VLLM_ROCM_USE_AITER: 1 + VLLM_USE_AITER_UNIFIED_ATTENTION: 1 + VLLM_ROCM_USE_AITER_MHA: 0 + HSA_NO_SCRATCH_RECLAIM: 1 + NCCL_MIN_NCHANNELS: 112 + USE_FASTSAFETENSOR: 1 + SAFETENSORS_FAST_GPU: 1 + +lm_eval: + model_args: + tokenized_requests: false + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 10240 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: isl1024-osl1024-conc4 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl1024-osl1024-conc8 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl1024-osl1024-conc16 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl1024-osl1024-conc32 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl1024-osl1024-conc64 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl1024-osl1024-conc128 + backend: openai + dataset: random + input_len: 1024 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128 + - name: isl8192-osl1024-conc4 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 4 + - name: isl8192-osl1024-conc8 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 8 + - name: isl8192-osl1024-conc16 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 16 + - name: isl8192-osl1024-conc32 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 200 + max_concurrency: 32 + - name: isl8192-osl1024-conc64 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 320 + max_concurrency: 64 + - name: isl8192-osl1024-conc128 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 640 + max_concurrency: 128