From e462fb768d6db55cd956e28a2656264acee4bc89 Mon Sep 17 00:00:00 2001 From: Liang Juhao Date: Mon, 3 Aug 2026 04:09:16 +0000 Subject: [PATCH] feat: add Suite H (SWA) to leaderboard UI Adapt all frontend surfaces to the new suite: - suite-meta.js: editorial entry (title, tagline, workload, scenarios) for suite_H - data.js: SUITE_ORDER and SUITE_COLUMNS include suite_H - modal.js: suite_H result panel (reuses single-chip AB layout + SWA note) - home.js: distribution chart includes suite_H - suites.js: SWA finding, scenario appliesTo lists, sharegpt_swa_v1 dataset, roofline diagram dot - generate.py: suite_H distribution block (offline/online/interactive/sustained/burst) - CSS: --suite-H amber color + [data-suite=H] bindings across all view files - hardware-catalog.js: A800 associated with suite_H (has submitted results) Co-Authored-By: Claude --- leaderboard/generate.py | 16 +++++++++ leaderboard/site/assets/css/base.css | 2 ++ leaderboard/site/assets/css/chip-detail.css | 5 +++ leaderboard/site/assets/css/home.css | 1 + leaderboard/site/assets/css/modal.css | 1 + leaderboard/site/assets/css/rankings.css | 2 ++ leaderboard/site/assets/css/suites.css | 4 +++ leaderboard/site/assets/data/suite-meta.js | 33 +++++++++++++++++++ leaderboard/site/assets/js/data.js | 7 +++- .../site/assets/js/hardware-catalog.js | 2 +- leaderboard/site/assets/js/modal.js | 24 +++++++++++++- leaderboard/site/assets/js/views/home.js | 2 +- leaderboard/site/assets/js/views/rankings.js | 2 +- leaderboard/site/assets/js/views/suites.js | 33 +++++++++++++++---- 14 files changed, 122 insertions(+), 12 deletions(-) diff --git a/leaderboard/generate.py b/leaderboard/generate.py index c817648e..261a1f1a 100644 --- a/leaderboard/generate.py +++ b/leaderboard/generate.py @@ -617,6 +617,22 @@ def _offline_reliability(rows): "runtime_metrics": rm if rm else None, } + if suite == "suite_H": + rows = _offline_rows() + return { + "type": "suite_H", + "offline": { + "labels": _concurrency_labels(rows), + "throughput": [r.get("throughput_tokens_per_sec") for r in rows], + "memory_gb": [r.get("peak_memory_gb") for r in rows], + "throughput_reliability": _offline_reliability(rows), + }, + "online": _online_block(), + "interactive": _interactive_block(), + "sustained": _sustained_block(), + "burst": _burst_block(), + } + if metrics.get("sustained"): sustained = metrics.get("sustained", {}) samples = sustained.get("samples", []) diff --git a/leaderboard/site/assets/css/base.css b/leaderboard/site/assets/css/base.css index 0ed4f04a..33a576a7 100644 --- a/leaderboard/site/assets/css/base.css +++ b/leaderboard/site/assets/css/base.css @@ -46,6 +46,7 @@ --suite-E: #16a34a; /* emerald — Scaling efficiency */ --suite-F: #db2777; /* rose — Edge / low-power */ --suite-G: #6366f1; /* indigo — Sustained load */ + --suite-H: #f59e0b; /* amber — Sliding Window Attention */ /* Vendor colour table now lives in data.js (`VENDOR_COLORS`) and is * injected at runtime as `[data-vendor=""] { --vendor-color: ... }` @@ -113,6 +114,7 @@ --suite-E: #15803d; --suite-F: #be185d; --suite-G: #4338ca; + --suite-H: #d97706; --shadow-card: 0 1px 2px rgba(15, 23, 42, 0.05), 0 1px 3px rgba(15, 23, 42, 0.04); --shadow-lift: 0 8px 24px -10px rgba(15, 23, 42, 0.14); diff --git a/leaderboard/site/assets/css/chip-detail.css b/leaderboard/site/assets/css/chip-detail.css index 5ce58bd3..d2096fef 100644 --- a/leaderboard/site/assets/css/chip-detail.css +++ b/leaderboard/site/assets/css/chip-detail.css @@ -52,6 +52,7 @@ .chip-suite-card[data-suite="E"] { --suite-color: var(--suite-E); } .chip-suite-card[data-suite="F"] { --suite-color: var(--suite-F); } .chip-suite-card[data-suite="G"] { --suite-color: var(--suite-G); } +.chip-suite-card[data-suite="H"] { --suite-color: var(--suite-H); } a.chip-suite-card { cursor: pointer; } a.chip-suite-card:hover { @@ -287,6 +288,7 @@ a.chip-suite-card:hover .chip-suite-hint { .chip-fp-row[data-suite="E"] { --suite-color: var(--suite-E); } .chip-fp-row[data-suite="F"] { --suite-color: var(--suite-F); } .chip-fp-row[data-suite="G"] { --suite-color: var(--suite-G); } +.chip-fp-row[data-suite="H"] { --suite-color: var(--suite-H); } .chip-fp-letter { width: 22px; height: 22px; @@ -404,6 +406,7 @@ a.chip-suite-card:hover .chip-suite-hint { .chip-scl-row[data-suite="E"] { --suite-color: var(--suite-E); } .chip-scl-row[data-suite="F"] { --suite-color: var(--suite-F); } .chip-scl-row[data-suite="G"] { --suite-color: var(--suite-G); } +.chip-scl-row[data-suite="H"] { --suite-color: var(--suite-H); } .chip-scl-row-letter { width: 22px; height: 22px; @@ -501,6 +504,7 @@ a.chip-suite-card:hover .chip-suite-hint { .chip-runs tbody tr[data-suite="E"] { --suite-color: var(--suite-E); } .chip-runs tbody tr[data-suite="F"] { --suite-color: var(--suite-F); } .chip-runs tbody tr[data-suite="G"] { --suite-color: var(--suite-G); } +.chip-runs tbody tr[data-suite="H"] { --suite-color: var(--suite-H); } .chip-runs-suite { display: inline-flex; @@ -645,6 +649,7 @@ a.chip-suite-card:hover .chip-suite-hint { .chip-peer-suite[data-suite="E"] { --suite-color: var(--suite-E); } .chip-peer-suite[data-suite="F"] { --suite-color: var(--suite-F); } .chip-peer-suite[data-suite="G"] { --suite-color: var(--suite-G); } +.chip-peer-suite[data-suite="H"] { --suite-color: var(--suite-H); } .chip-peer-runs { font-variant-numeric: tabular-nums; white-space: nowrap; } /* ── Responsive ───────────────────────────────────────────── */ diff --git a/leaderboard/site/assets/css/home.css b/leaderboard/site/assets/css/home.css index f454868d..b91bed00 100644 --- a/leaderboard/site/assets/css/home.css +++ b/leaderboard/site/assets/css/home.css @@ -13,6 +13,7 @@ .suite-card[data-suite="E"], .lb-row[data-suite="E"], .vendor-suite-pill[data-suite="E"] { --suite-color: var(--suite-E); } .suite-card[data-suite="F"], .lb-row[data-suite="F"], .vendor-suite-pill[data-suite="F"] { --suite-color: var(--suite-F); } .suite-card[data-suite="G"], .lb-row[data-suite="G"], .vendor-suite-pill[data-suite="G"] { --suite-color: var(--suite-G); } +.suite-card[data-suite="H"], .lb-row[data-suite="H"], .vendor-suite-pill[data-suite="H"] { --suite-color: var(--suite-H); } /* Default fallback color */ .suite-card, diff --git a/leaderboard/site/assets/css/modal.css b/leaderboard/site/assets/css/modal.css index 26d51c7f..537e2c61 100644 --- a/leaderboard/site/assets/css/modal.css +++ b/leaderboard/site/assets/css/modal.css @@ -21,6 +21,7 @@ .modal-shell[data-suite="E"] { --suite-color: var(--suite-E); } .modal-shell[data-suite="F"] { --suite-color: var(--suite-F); } .modal-shell[data-suite="G"] { --suite-color: var(--suite-G); } +.modal-shell[data-suite="H"] { --suite-color: var(--suite-H); } .modal-shell { --suite-color: var(--accent-2); } /* ── Backdrop ──────────────────────────────────────────────── */ diff --git a/leaderboard/site/assets/css/rankings.css b/leaderboard/site/assets/css/rankings.css index b7ef565c..b0bc0b1e 100644 --- a/leaderboard/site/assets/css/rankings.css +++ b/leaderboard/site/assets/css/rankings.css @@ -21,6 +21,7 @@ .rk-hero[data-suite="E"], .data-table[data-suite="E"], .rk-chip-focus[data-suite="E"] { --suite-color: var(--suite-E); } .rk-hero[data-suite="F"], .data-table[data-suite="F"], .rk-chip-focus[data-suite="F"] { --suite-color: var(--suite-F); } .rk-hero[data-suite="G"], .data-table[data-suite="G"], .rk-chip-focus[data-suite="G"] { --suite-color: var(--suite-G); } +.rk-hero[data-suite="H"], .data-table[data-suite="H"], .rk-chip-focus[data-suite="H"] { --suite-color: var(--suite-H); } .rk-suite-pill[data-suite="suite_A"] { --suite-color: var(--suite-A); } .rk-suite-pill[data-suite="suite_B"] { --suite-color: var(--suite-B); } @@ -803,6 +804,7 @@ .cmp-table[data-suite="E"] { --suite-color: var(--suite-E); } .cmp-table[data-suite="F"] { --suite-color: var(--suite-F); } .cmp-table[data-suite="G"] { --suite-color: var(--suite-G); } +.cmp-table[data-suite="H"] { --suite-color: var(--suite-H); } .cmp-table thead th { background: var(--bg-elev); diff --git a/leaderboard/site/assets/css/suites.css b/leaderboard/site/assets/css/suites.css index 24006fcc..6a2ec5c7 100644 --- a/leaderboard/site/assets/css/suites.css +++ b/leaderboard/site/assets/css/suites.css @@ -30,6 +30,10 @@ .inversion-card[data-suite="G"], .scn-card .scn-suite-letter[data-suite="G"], .rfl-dot[data-suite="G"] { --suite-color: var(--suite-G); } +.suite-spec[data-suite="H"], +.inversion-card[data-suite="H"], +.scn-card .scn-suite-letter[data-suite="H"], +.rfl-dot[data-suite="H"] { --suite-color: var(--suite-H); } .suite-spec, .inversion-card, .scn-card .scn-suite-letter, diff --git a/leaderboard/site/assets/data/suite-meta.js b/leaderboard/site/assets/data/suite-meta.js index 94dcf874..25fbb80d 100644 --- a/leaderboard/site/assets/data/suite-meta.js +++ b/leaderboard/site/assets/data/suite-meta.js @@ -234,4 +234,37 @@ export const SUITE_META = { metric: { key: "sustained_throughput", label: "sustained throughput", direction: "desc", unit: "tokens/sec" } }, ], }, + suite_H: { + letter: "H", + chartPurpose: "Sliding Window Attention (SWA)", + title: "Sliding Window Attention (SWA)", + tagline: "Long context, capped KV cache — architecture-driven intensity shift.", + description: + "SWA as the architectural variable. Mistral-7B runs every layer with a sliding window (4,096 tokens), so at 10K-context prompts the KV cache is bounded by the window rather than the sequence length — moving long-context decode off the KV-capacity cliff. Frameworks vary in whether they honor SWA-aware KV allocation, which makes the suite a check on how much of the theoretical memory win the software stack actually delivers.", + primary: { key: "offline_throughput", label: "tokens/sec", direction: "desc", unit: "tokens/sec" }, + workload: { + model: "mistralai/Mistral-7B-Instruct-v0.1", + chips: "1", + precision: "BF16", + dataset: "sharegpt_swa_v1", + inputTokens: "~10K", + outputTokens: "~256", + }, + scenarios: [ + { name: "accuracy", isExtra: false, + desc: "MMLU gate against the Mistral-7B baseline." }, + { name: "offline", isExtra: false, + desc: "Throughput at 10K-token prompts (window = 4,096), where SWA bounds KV growth.", + metric: { key: "offline_throughput", label: "tokens/sec", direction: "desc", unit: "tokens/sec" } }, + { name: "online", isExtra: false, + desc: "Max QPS under the 500 ms p99 TTFT SLA on long-context SWA serving.", + metric: { key: "online_max_qps", label: "queries/sec", direction: "desc", unit: "queries/sec" } }, + { name: "interactive", isExtra: true, + desc: "Single-stream TTFT at 10K-context prefill." }, + { name: "sustained", isExtra: true, + desc: "30 min sustained SWA load." }, + { name: "burst", isExtra: true, + desc: "Burst tolerance on long-context KV pressure." }, + ], + }, }; diff --git a/leaderboard/site/assets/js/data.js b/leaderboard/site/assets/js/data.js index 6a3a1ef5..83238a73 100644 --- a/leaderboard/site/assets/js/data.js +++ b/leaderboard/site/assets/js/data.js @@ -55,7 +55,7 @@ export function formatPrimary(value, suiteId) { return p.unit ? `${num} ${p.unit}` : num; } -export const SUITE_ORDER = ["suite_A", "suite_B", "suite_C", "suite_D", "suite_E", "suite_F", "suite_G"]; +export const SUITE_ORDER = ["suite_A", "suite_B", "suite_C", "suite_D", "suite_E", "suite_F", "suite_G", "suite_H"]; /** Line 1 for charts: "Suite A" */ export function suiteChartHead(suiteId) { @@ -207,6 +207,11 @@ export const SUITE_COLUMNS = { { key: "sustained_throughput", label: "Sustained", unit: "tok/s", direction: "desc", primary: true }, { key: "offline_throughput", label: "Offline", unit: "tok/s", direction: "desc" }, ], + suite_H: [ + { key: "offline_throughput", label: "Offline", unit: "tok/s", direction: "desc", primary: true }, + { key: "online_max_qps", label: "Online QPS", unit: "qps", direction: "desc" }, + { key: "interactive_ttft_p99", label: "TTFT p99", unit: "ms", direction: "asc", decimals: 0 }, + ], }; // Format a numeric value against a column spec. Returns null for diff --git a/leaderboard/site/assets/js/hardware-catalog.js b/leaderboard/site/assets/js/hardware-catalog.js index 01cfba56..1a6ba66e 100644 --- a/leaderboard/site/assets/js/hardware-catalog.js +++ b/leaderboard/site/assets/js/hardware-catalog.js @@ -42,7 +42,7 @@ export const CHIPS_BY_VENDOR = { { name: "H800 80GB", memoryGb: 80, tier: "datacenter", suites: ["suite_A", "suite_B"] }, { name: "A100 80GB", memoryGb: 80, tier: "datacenter", suites: ["suite_A", "suite_B"] }, { name: "A100 40GB", memoryGb: 40, tier: "datacenter", suites: ["suite_A"] }, - { name: "A800 80GB", memoryGb: 80, tier: "datacenter", suites: ["suite_A"] }, + { name: "A800 80GB", memoryGb: 80, tier: "datacenter", suites: ["suite_A", "suite_H"] }, { name: "L40S", memoryGb: 48, tier: "datacenter", badge: "Cost-efficient inference", suites: ["suite_A", "suite_F"] }, { name: "L4", memoryGb: 24, tier: "datacenter", badge: "Edge inference", suites: ["suite_F"] }, { name: "A10", memoryGb: 24, tier: "datacenter", suites: ["suite_F"] }, diff --git a/leaderboard/site/assets/js/modal.js b/leaderboard/site/assets/js/modal.js index bd1a7046..c7011b77 100644 --- a/leaderboard/site/assets/js/modal.js +++ b/leaderboard/site/assets/js/modal.js @@ -887,6 +887,7 @@ function _renderViz(row, panel) { case "suite_E": _renderSuiteE (panel, row, viz); break; case "suite_F": _renderSuiteAB(panel, row, viz, "purple"); break; case "suite_G": _renderSuiteG (panel, row, viz); break; + case "suite_H": _renderSuiteH (panel, row, viz); break; case "sustained": _renderSustained(panel, row, viz); break; default: panel.innerHTML = `
Visualization for type "${esc(viz.type || "unknown")}" is not yet supported.
`; @@ -1035,7 +1036,9 @@ function _fmtKMs(v) { // Used for suite_A and suite_F (just changes the accent color). function _renderSuiteAB(el, row, viz, accent) { const C = chartColors(); - const accentHex = accent === "purple" ? C.purple : C.blue; + const accentHex = accent === "purple" ? C.purple + : accent === "amber" ? C.amber + : C.blue; if (viz.offline && viz.offline.labels && viz.offline.labels.length) { el.appendChild(_section("Offline — throughput by concurrency")); @@ -1486,6 +1489,25 @@ function _renderSuiteG(el, row, viz) { } } +function _renderSuiteH(el, row, viz) { + // SWA long-context suite: reuse the single-chip AB layout + // (offline / online / interactive / burst), then surface the + // sliding-window angle explicitly. + _renderSuiteAB(el, row, viz, "amber"); + + el.appendChild(_section("Sliding Window Attention")); + el.appendChild(_statChips([ + { label: "Window size", value: "4,096 tokens" }, + { label: "Prompt length", value: row.offline_throughput != null ? "10K tokens" : null }, + { label: "KV cache bound", value: "window, not context" }, + ])); + el.appendChild(document.createElement("p").appendChild( + document.createTextNode( + "Mistral-7B applies a 4,096-token sliding window on every layer, so at 10K-context prompts the KV cache is capped by the window. Whether a framework actually honors SWA-aware KV allocation determines how much of that memory win shows up in the concurrency ceiling." + ) + ).parentNode); +} + function _renderSustained(el, row, viz) { const C = chartColors(); const dur = viz.duration_minutes != null ? viz.duration_minutes : "?"; diff --git a/leaderboard/site/assets/js/views/home.js b/leaderboard/site/assets/js/views/home.js index 38020c99..11fcf409 100644 --- a/leaderboard/site/assets/js/views/home.js +++ b/leaderboard/site/assets/js/views/home.js @@ -37,7 +37,7 @@ const DIST_METRICS = { speculative: {label:'Speculative Throughput',unit:'tok/s',get:function(s){var o=s.scenarios&&s.scenarios.speculative;return o&&o.is_valid?o.throughput:null;}}, }; const DIST_VCOLS = {'NVIDIA':'#1b7a3d','Huawei':'#c2410c','AMD':'#be185d','Google':'#1d4ed8','Apple':'#78716c','Moore Threads':'#7c3aed'}; -const DIST_SUITES = ['suite_A','suite_B','suite_C','suite_D','suite_E','suite_F','suite_G']; +const DIST_SUITES = ['suite_A','suite_B','suite_C','suite_D','suite_E','suite_F','suite_G','suite_H']; function distVc(v){return DIST_VCOLS[v]||'#7c3aed';} function distFm(v){if(v===null||v===undefined)return'—';if(v>=10000)return(v/1000).toFixed(0)+'k';if(v>=1000)return(v/1000).toFixed(1)+'k';if(v>=1)return v.toFixed(1);return v.toFixed(4);} function distHs(s){var h=5381;for(var i=0;i`, }; -// Roofline mini-diagram. Coordinates are hand-tuned so the seven dots +// Roofline mini-diagram. Coordinates are hand-tuned so the eight dots // land at roughly the right region of the spectrum for each suite. const ROOFLINE_POINTS = [ { letter: "F", x: 68, y: 168, label: "right" }, // far left, bandwidth-bound, smallest model @@ -306,6 +324,7 @@ const ROOFLINE_POINTS = [ { letter: "B", x: 138, y: 96, label: "left" }, // 70B multi-chip { letter: "C", x: 168, y: 68, label: "left" }, // quantization, transition { letter: "E", x: 192, y: 56, label: "right" }, // scaling, near the knee + { letter: "H", x: 220, y: 62, label: "right" }, // SWA long-context, KV capped { letter: "D", x: 268, y: 56, label: "left" }, // compute-bound long context ];